From 7f2637db8c663520d3f33c46581ee4ae05d01e9d Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Wed, 26 Jun 2024 15:14:17 +0200 Subject: [PATCH 1/4] starting point --- s7_deployment/deployment_testing.md | 124 ++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 s7_deployment/deployment_testing.md diff --git a/s7_deployment/deployment_testing.md b/s7_deployment/deployment_testing.md new file mode 100644 index 000000000..e9bacc65f --- /dev/null +++ b/s7_deployment/deployment_testing.md @@ -0,0 +1,124 @@ +![Logo](../figures/icons/cloudrun.png){ align=right width="130"} + +# Deployment Testing + +https://cloud.google.com/architecture/application-deployment-and-testing-strategies + +The testing patterns discussed in this section are typically used to validate a service's reliability and stability over +a reasonable period under a realistic level of concurrency and load. + +In today's dynamic software development landscape, ensuring seamless and reliable application deployment is crucial for maintaining user satisfaction and operational efficiency. This module will introduce you to various deployment testing patterns, each designed to minimize risk, enhance performance, and ensure a smooth transition from development to production. Whether you're aiming for zero downtime, incremental feature releases, or robust rollback capabilities, understanding these strategies will empower you to implement effective deployment workflows. Join us as we explore canary deployments, blue-green deployments, feature toggles, and more, equipping you with the knowledge to optimize your deployment processes and deliver high-quality software with confidence. + + +## A/B testing + +### ❔ Exercises + +1. Geolocation + + ```python + from fastapi import FastAPI, Request, HTTPException + import geoip2.database + + app = FastAPI() + + # Load the GeoLite2 database + reader = geoip2.database.Reader('/path/to/GeoLite2-City.mmdb') + + def get_client_ip(request: Request) -> str: + if "X-Forwarded-For" in request.headers: + return request.headers["X-Forwarded-For"].split(",")[0] + if "X-Real-IP" in request.headers: + return request.headers["X-Real-IP"] + return request.client.host + + def get_geolocation(ip: str) -> dict: + try: + response = reader.city(ip) + return { + "ip": ip, + "city": response.city.name, + "region": response.subdivisions.most_specific.name, + "country": response.country.name, + "location": { + "latitude": response.location.latitude, + "longitude": response.location.longitude + } + } + except geoip2.errors.AddressNotFoundError: + raise HTTPException(status_code=404, detail="IP address not found in the database") + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @app.get("/geolocation") + async def geolocation(request: Request): + client_ip = get_client_ip(request) + geolocation_data = get_geolocation(client_ip) + return geolocation_data + ``` + +## Canary deployment + +### ❔ Exercises + +Follow [these](https://cloud.google.com/architecture/implementing-cloud-run-canary-deployments-git-branches-cloud-build) +instructions to implement a canary deployment using Git branches and Cloud Build. + +1. Use the following command + + ```bash + gcloud run services update-traffic + ``` + +## Shadow deployment + +### ❔ Exercises + +1. Google Run does not naturally support shadow deployments, because its loadbalancer requires that the traffic adds up + to 100%, and for shadow deployments, it would be 200%. To proper implement this you would need to use a kubernetes + cluster and use a service mesh like Istio. So instead we are going to implement a very simple load balancer ourself. + + 1. Create a new script called `loadbalancer.py` and add the following code + + ```python + import random + from fastapi import FastAPI, HTTPException + import requests + + app = FastAPI() + + services = { + "service1": "http://localhost:8000", + "service2": "http://localhost:8001" + } + + @app.get("/shadow") + async def shadow(): + service = random.choice(list(services.keys())) + response = requests.get(services[service] + "/shadow") + return { + "service": service, + "response": response.json() + } + ``` + + 2. Because the loadbalancer is just a simple Python script lets just deploy it to Cloud Functions instead of Cloud + Run. Create a new Cloud Function and deploy the script. + +## 🧠 Knowledge check + +1. Try to fill out the following table: + + Testing patter | Zero downtime | Real production traffic testing | Releasing to users based on conditions | Rollback duration | Releasing to users based on conditions | + --------------- | -------------- | -------------------------------- | -------------------------------------- | ----------------- | -------------------------------------- | + A/B testing | | | | | | + Canary deployment | | | | | | + Shadow deployment | | | | | | + + ??? success "Solution" + + Testing patter | Zero downtime | Real production traffic testing | Releasing to users based on conditions | Rollback duration | Releasing to users based on conditions | + --------------- | -------------- | -------------------------------- | -------------------------------------- | ----------------- | -------------------------------------- | + A/B testing | No | No | Yes | Short | No | + Canary deployment | Yes | Yes | Yes | Short | Yes | + Shadow deployment | Yes | No | Yes | Short | Yes | \ No newline at end of file From 81234f5a802a8801120f37f7bf6f6003c111eff9 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Thu, 27 Jun 2024 12:32:05 +0200 Subject: [PATCH 2/4] add figures --- figures/a_b_testing.svg | 710 +++++++++++ figures/a_b_testing_example.png | Bin 0 -> 44456 bytes figures/canary_deployment.svg | 2092 +++++++++++++++++++++++++++++++ figures/shadow_testing.svg | 570 +++++++++ 4 files changed, 3372 insertions(+) create mode 100644 figures/a_b_testing.svg create mode 100644 figures/a_b_testing_example.png create mode 100644 figures/canary_deployment.svg create mode 100644 figures/shadow_testing.svg diff --git a/figures/a_b_testing.svg b/figures/a_b_testing.svg new file mode 100644 index 000000000..bb407beb8 --- /dev/null +++ b/figures/a_b_testing.svg @@ -0,0 +1,710 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.1 + 2020-01-16 10:05:46 +0000 + + + Diagram 6 + + Layer 1 + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + path arrow - grey primary + + + + + + diff --git a/figures/a_b_testing_example.png b/figures/a_b_testing_example.png new file mode 100644 index 0000000000000000000000000000000000000000..4cfe49ef044a8dd18c8046ca5e6b91d19cb1739e GIT binary patch literal 44456 zcmd431yogE`!2c_MM63yB}5vLZb3p?KvF{K?#2x&C=E(C0#cijj!g&%NH?4Au1z=W zJNbRz8RLHE{LdZtj{mr0oHf)1d(So3%=dlX=Xs~WuT|u5uqdz~5D1R^D`|BI1T7u{ zx%(L7F8Cx(6{{QEFdbj%xl14Dk_|_9KUuT#uKYtRW}<8y+`z_~X)O5~Us70c z=K0F?4SD3$4{ml&th4K@?C#x|)7i%BzGMNO7;ahcdI)5a1Xt#tA6}LIueXoH*tZ}4 z=k32$5c035|Fgz_|Mq`o#s6;HKm-Q!-7}3a?&Mzc@(EL2^RVDdOJ4FYCZY}}d!z3& z_SL#YxZqRF&+m9C!etry5A*IrKAInoY84bRTU=Ua4O;S6sTQVX;KScM>C-EhQ zd^65iA(qjtw;Zf<+bWBs7I9e(WABbpmwNTu(ecBFt8a8KcWbsdxwuL~jlRQRY<}k} zYw3P{LcaCCehn-*u9rMOM{jR$KiNVcXXT^sUR|p(Kp<;TW?s#a684i7EKfz?zka>e z)Kp;UaFUdi+~42V#-ntf+Nt*O^sHHtdw*VLR-j!ZoR(_v<_%+a*^RfOqocmQzIBuD zscO>l(vm;LCq@6~?XR&M4ZGtwE~kc7jcUF)Qk^a@cWR{h>Zw+9l-nSRMI0JpuOiyY*GI5ew&YrRBcnMZ?Gjup zEGgg@WP_WN86vc#k$wMQ%xeyHs3csXflUKB%EDW*v~m8k3AY?4H!93!owkdP25 zugFmSUBA`G@9w?y0r>19zL|ak>gdS6tjNg?p~$HcCfdnQ*LkY>J^4H z);Hq?LN(3f_N@|+%Icx}_wL!wl)jAmY2N7fhYzuwDmWOWrMU5_kiC2&?Bec6w&+Y| zZM;W|)&Zkm0s?GjtBTIGdvU3s|6D)Xh&vt)jG*GC2+!lPe3D>{(Zu7iI~^yZBlSwg zY-Kq&&3ZMt3AGEq{RG2gd(x(OkQe9pK14gGU;Mhlj35qQU0wa4XC%M(!(ndvhoLfz zg40nG*~s>Hk1A-w+FDQ}2*l~;;F2-Lz3ihYWFw`Z>S(}n=_`&V8QFVGy1Cx;5@sxG97r*5Nz=)sOoRlt)GYLL zMglsuIa>enF(a0 zFLESbgY3*E6sbMT`-ia1V@~`H)#ElPwx}78?3m-vPct}u<2199LeurJAp^yNR1u29 z+6csGt&Pkm8B~Z1>?Hl5+}vC^>B`a$^b<=Jhv(}ZT1;FVnYKH>N!r5W<cTIC&LkJsFG9X(@x9x7S3RBe5r3%2=M?*juThb#x-dxHd&cPIa|qf;wyr$_hUqbF zqi4D9TVuUqY?-bj$B3*&2n(s6+FD;KpND?L=GGdb(guNDFNuGzI_%OqDQ*ZZ@XydY zIXT(azLa!)^~%%p?EdWI5MvLIhaJ{QV-0e6|BT3%_bDx^=8OUP$A0Dm6v}m#7H{aa zvWU0W5ka}uURw`=?A*;h8buhw&l>AI6uQ#gGx}AM-S;U!AjD3|AELjZs^~)8>EFG( z#;{5=Q)kd~|5wZqb+2by9D8^5tn2V6v9&xYnuzTC;)GkI!3zaWlgg!dMr`^Di2jTfWZM#m2AJH}zMOWh^)&Y;XvjZ@IC z0q-+yBPdBWB%GdsK~X^gw{jg5!=xkJXJI8i*(;4rR8;iYGsoetI2?jx0}XPKUWaS& ztqXL>NB5^p{QM#!BDpen-)j$x;Wl8T^pPr5!$|dwUI=c$x(x>RynSobtJN^V~up*O2%=^k>-#BeeO#>{m_>pjv_InQ= zILwsEXpTxi%O7Lli60HD8ToFg2aIOtRyVys*z`Sj;_{}y?)?kqET?n~= zZ+&D5vdV5MA?JD{`H8&uV&_won2i>8)XHy3e|mZb0lV>mp2mxt2Q&%i3xQFL84*7; z7dAQ1URG9C4jXxyoku+hQ+IB3Z*kH`{P|PTc)@zKn3iEysW%5*OFchaRr9_`(Eu&c^$HohRzabDBPDM=eHdvihc?#{Uf(KkUcqTYio1A!`{=wHJBPHDU)Up4Gt)8keQ4rmd|lzpI5nWaD*JT~2##35}5RvWcp=*MiT$T`;FB888@q zH-$+ibJe?pkBV$jW{KQP7K{tdY7aCgc6!FfB8AL$`dL7B`UnKe$I z!nmV5T`{$!S=!2R?#yG+bvwJ*Q~Nhk02ilFYBrpco2$fKgFMk3)zsV)uoJGf5KT+H z9{4F=_SQ!>VklT(PV_j@!V!5FvOeznwtV%=TDpgck%6XKsoh$i%jG7p!5uWdI(F?d zaDC^w>r;t~ag#hb$F)I%)i%RUaueFV6d{{cB>{dkL;O{MSRDs`FqLqVZAb|bwl=yxp>*g;6Prqf^7V7+l`#ql)pl-Sld*fAw~@WgFQM`6 z=0MRq#R|RlZm!aGI({#4WcK}q$9$vEVFvn(_}YAsEF(#Go1ku;Z3<` zX<*<3YtmbEN8o;xr`8cc?cz`^xUmpuTEX4zd-w41^ICtrHWT(i{+12n;ss)DsfvF`4z=za+Y!uL%#n+56d zTuna%&8P%UzET!yu~|QYOu`X4svbT8$A^c>$NL@1^xK7rJOagUJy<-HhbyM+8z&YV zf5hEkA=Q7jHHW%c-71+$p=N;s*N%pUmTIIl*KnCyJ!>7Vc#o6)^iS|w8kF?uBgl^i zJb@3W84r78!JPLRwULq08dc=VhhJx zbAH=3PJJaYv9UA0TSf6~3dI-4i2D*ZL7V6w-L2OmsEwfeU#8##1;ax(&#hw)*<@s7 zPUl?uS0;O(NL*K(<)*)$88|@Wd8lh3wC#SG_nl$JCl*5Y$C7tfN%i#Lz}ax{7-0a+ zouANKemZP2uHE5WyiIDzWc=5!Uu^>MxL8=6kjA~{E*G;6vbkGEloS+S@u>9-)trP* z<4V)DixiZUq#IshzhghaI#@jHhl>HEla?ktDg^GUy@@Vf4D56P1P9wZ{qH`rLn=i2 z#SX)!z-P;-n;VZMq{G4iK)VL37>XDfN9PvlZTED+I>4TYPHQx7!>Z;Qvwc1TMK*vD zh>N>fO7)m*;apo==}i~F)5L@9{H-c_izw7xI%&JOyo^j296XN&7^Gj~{3plu!S_P; z(^(v{VyXl+N10y}w*JL+u)4a*nZSXgbYzpnLQZmuk+WMDw$+N~P_y6Ou1#*j(%I=&S!ri6cWP>Sc7C3L^HK2`sgF#8gkR%r ze*&!ZqA^4qX6yM$031YwQw5^~Z{5;C3tc|JCHMI(s_K%Dv^yu9a_}MKM|eR&!R79h!`rM9!wOURXp-Jk zF?i9Aua(n&+L0?S8W^m(^UZ=1y`==W%UWOl^@UJVnbT7H=g*({b?yRNlh2H$2kfF* zy5ET|)F5?=jFOBteWX11kWGTi$Z(=vC22$aX0KJzZ%Pl;Rn@)V*mE`SKbCMU>VD<9 zmivyUJxA%+d_w$kIeM%{WhFvn7yn+yeiIDVcyqC3SB}r` zFstu*!ApS#47Dz@;WTYbm~3yxd8?>t3(9Y2omb*ezOn~8^iM^6&z?oU|?uGEGq+c0!En-;0XU*56FTcKtiHAcw8*grWPIp{bbz7Ofd=$4W_d@^GH{(^FbQ=`T12~S>lGZxtsb7 z4Zme*QCG0Ix}57P+3x@Q!w$|&Zo(yo5>2`XsgO(-aEN=mP=$&a@|cAp&SR;ks;jR7 zkZD3uE8qQQT;dv5YNi-pHIy0n@X_K`3xF}BC)lRfUvWHR$RHK9je5gc>_)Q`+I-yY z7pF+%RaYR7Mb$OYuLA9os_F$Hyj#fdnY}yI?;@Ws?n6hahx0T_PX!8vn#;<AvUg@(5h5H9L@XVsb4geB7oO*cSblE2H{hRiQDq34wjZLrXU}ymckK64zZ1s!#wo#E$6YZO;)^O%)oZP~07bS5Mfw`>q`Ji`#IusT-;d@}~nLNIR#luI`|$ zY)t3i_b0UE=^>xl>Drxa9R&r2kkHUUQ3%Tk)~I*CK0xTd4?isVg<@f+qNc{UPYw@O zENLJEvW14VWw9zLr>_*88$V{ukFXq#4(!kSmQri%C-TH?ZjR|VTUzEtMyf9P+1S`% ztE4VY%CG$;=Sfr=92~TygA9_z_Ma7#(f_~zZ4egKjWg4=zqtg^NQPhq z-R_ZDSO)R!7th+qjZhbey}i9vzn>KqN3^K32cOw1%*aDQ+t<%n#-^s=V6Hi#3UOv2 zp6huTCv2EXc^n z7;Z=8_x*RWcWxj4*ZBNTH+A;guRwJ6f4lwn3jX`)|ETfb_5UAQ@qaY+g$?2BdtflA z{n@)!#g0e3P{fZOZ470Sn`q>!h#}L(FV-2b;>(|V@I}lDPQ%pMhN8e=vzEnNDEm~Sy`sK@)BdV8yxNcTL z+}zv?3k%Amnam0*Dk`}usjvGWkY3C(d#Zn)2)<><<~bkWQaK_An^fXf>g($}qo0~Q zCx!fevKvkCKBKoqhrMDZ%THPd^09mBDMQO$b@m^_(gF7&mKMm9T`Y`*)U^3%B7(6zh8s8*eVg2 zc0{9RuaJx3{;K{_F~8HL0lW;tnf3K6I^eFHAwP8>h|jm}fIuo9niKyv`;lHZS?2~U z#Y^xq$VbxOI_&!VV1I>&hW_)(?Gxatp`TZId;7!xi!J`I4gcRO`0uCx({}vpw}DbX z=|Vn6{4p*ax8;vP1DJ+}hQ`p)&|a6=EX?3aoeDQFlbO1#kR?0b*1$d=d;ML}-S9sg z$4^2sAL|o5*x}>jOUg;?U5%)=2`laD>QX%18RjhlYxr!p0^7~a4L5LUs6>0X;^W7U z-DfTXzD<8|aB!HSJ1uz)uZE@viryBDf#LLT9@y{F8?FEjKyg_1k;&}rtmWK})XSIt zFj)LbcckplqJ39KN8s8yZc}bj&ZCE4alI;BSUf`1Ll|UVFjzgwWVTLPjgzT;+LqL7 z9Lb%;!G<`{H&@& zS!goOofjQSrD3oA8Tn^bKXEI&3zBE}9oaWqzQaxKa*PBL$7$|3SK}EyAD?E^r};!4 zi>Rn5@8eCkS%E{*^yuj5gapq8;q?A7kSRs|!L z1g~`^MEeB}cL%9DD(7*bnas{q9pv!6@ayCBN$l;K#~8)pDX+LxujgGTvKyYU_*7vF525x7tOwdMN2)@FXi~9mTsyc8O@!}QwgU6+Vb?% zZhCUNE%SyIr{S&ZyE|lgd|mrggKuyGZy*p(i;}HkxJu-6z)b-1TvM<(+H4>1N^_~2_|8HM~DD;SKQmzS8Bn30i@_om7T)kdyC{Dr&y zu--S>@On)@it&0i?1C$g=FYL-)~D9dz-MX-9HiaF25|eD1@!juOqtN)+Eofw)&4Oy zS%nP{W|ozg%Y~6=z&3Yh;6A6j*SP}DTAF-ZTwF{{?b-TLEZ0ZFs-dBw4D|H7bz8;Q zh1peAWA@dk!876hL!XoFjj3vfLt2Cs3kMq;aF%h5@|nykX`&OYtyw~p{fAd^9B&DU zh_0`$I8DBpg%#_XdGBP(K{c75 zgNv&lfk3pjwq|qb+1u|<*Shd2Q;N9TqHeCSv9Rvmy_-Udn~it{eqHNL_Sh<_PJ`n- zdGaJJEp68uBR$6o*@SBJveAT*W1V@8z9<{50f(lyzbgqYP!IVSzZpv+7&JK65Eu%-A z^jNPiGjPjV!@88jCFv1$bDE=Ch#$d*=4kJ&-HLiUF8iC*luXJNzcC`x^4kfr>uU-n z)x5GiEOW8Nm=rXL!wJ8mvY4cIaNIH{TC)=oNm)5R3$+*m4vceu;0F&UP0UOdun|W> zRBsMwF3T@qWd{^&@0AXAi8t5I;Y4^b)talZQ=(E*ckgxec?9IVNiF7FLQ$Vul)gPHDtH)D#wmse7-bdZ44D-<&N)K2=S7Lu_`o zx6t|(#|L$N>i_vO78aJU$L{vhQuO!l-+@^<>=n?~)NDZAT!xiCP4lE@Wc-tT)Jh(0 zrc28+EN5B(g8?pX!WI`3vsABWWo0EP=lt>#JONZ8K>Wdgk?T?@pP(Wp9{Jon*qbZ> zG%Dabt_h?q`cepFN`dWHAX0Da(YfPj^oYhh+)2CV;M3_3bGz|j`x=VM}EFmrJs(%-Bt zEt!;009WIgcDlbmQDyhGKTXU4y4Ue1n5d*~lQp^bq!mvhj9O&Ox^hz$r>M9%j*$hB zjSG&z@^_YPudS`^>=*#U3WyaWqY~P7Ko?jik&=_^Xlo;YQP|wH_HTZSaqE28yFq~e z`7xku1>%Bd#osp8-!ijqSV2K~ zdAW&+2?GN|cUKn}iuCmKPE&v=Y)*G)I>H`hMLNaB$9I=)@7!KRR(b%9y?ggAp3^`< zQIQ%qFfuYSK3?0vAhpeSP;s^rBLiko=lV#(xBkW8J~Dm+0F0L zgI_e5hM9aey&)8+<#)(UyE%@@Z@TkS9AlA7jE(l)A?ek--RxyP?wHl~7yVzezQh`0 z&cElA`X9LfVy#LR{x5WQ7c}nc+6Zh58>-I|aAx<50a0uFlCZ*Cbip-$W93U@n*kMW`t zJ54YA2`pcpz#d7Ep&y6u{IQ8!*@`?-2qC#|*gu!q{21%=Zz7o|=-;*ZWW6LG|FLH4 z>CDG|c#>T#9a+X8GDgFi;jG;E>ByKxT>RDI6Mtq5Dl?dDIJ!LkJ;`}qO*Xckq9~k3 z8gwX{X9NpO_I~R`HAZZ59Q8&DVQ+q~_#MeFUx&B+X7zgre603AX_(EbseO`_{}HAp z86z!;p9iz1dYX2PH~JWo>;ZCH7|KW zN`V`Cmya{{5~qK#MyaO8uGMwyi}G1Sewj!Ft2{g;X3Xk*9Q-EwzUmZAyNP{gjDwST ze3&u#ZZ3>9E0g(|rrGCaQ+!3z#+n|{Qlrs0C@=e;Xy>(5sLQM@{j=_`>D4jZ6M6>T zo|gnKjjKb#(4?e;5}l*KM!PGQ$UIhraxyZV8{dq6qW`TV*g1+3^YaUqVP4sM_k($Y zF_z!aWJg#lf;rYB%)#}JRm%pfjQ$L#hsYCeAIn&h6?ctSKgM6P9g2$^-W?@>N^uYN zoK-&R@m@?mtXs5@JS-_aq8d)6djxWvUqdC%Ie0jqAYQ*>ARc-TO zlb#VJz_QqWew;!XOK$S$(W9p-DT1!+uR*vw+u(Cc;_U1+SPkcZKTcEvfWJc+`8D{V zZqog3y!7<+WMyRmO-stDFKE^kV^h-LzR>bT%;yBSZi^=0YbK?H`IQwjUJ78rL4X9J zHwwQ^uy2dBwiqHs7aawG&2V&lMz;(A01{C$2RyZghQ{g6lz{E{FH9n)LY-0|PZ0oK z&8X5^6PP@U-lXGpN(X(HUOpc49&%yV_4_Bqoy8kyouQ;0U7ej1a1RH-@_;5C5!3)i z4%qIVo*ocEj*gDnO_Y6Es-YHfmzI+&d0QQ^cD}f19I*s+#N;_T4FwQY9syVctFHEt zmHj$u&B4J@R8q3$0ki6%gp0FnfFuHfL{UO&hKlWm~k z0@RTe1DMvpz(8PdkrU-|YB3<1l#!L?)cN(>mLEA;$;QN#^W(>l;$lu_W=r4>nWDkK zDg%!IAR)k}AOHoO4zM@ClH%Zu@+!xz@L2S1&o+oWe*6wmYyg9q zTUY>l6&n|~J6RdZ?S6H(|NHlEKAW-E3gu;G09!jdJ0Bb#+E`lyFLTsnKT#$lvCPcM zs#k6v-nb9=pKzH5=p|73stBZj={HaaU!ykRWMw@#IwF=C93JN3;!-7U10}fIK?Txk zx3g+#X=!g??5oghU%j=I1sX1Kwf7R3>T27GsJOT)52T?aer9H7A6$$ndSe<%kv(F| z&-J#7osA8smh~VU2TKgyUvdr-v2?OKLrF?n;#F*I)hbOg)H&8?C-#9NF;)bp2Wx;VQ3C-tP{S1o-&B ze6PI~HX5OqKx_p<5t*hYSPS40;3*VMT4U6!MMKKmE?s4rRRtySc_T_BV(bs$3a=G$ zw>xY07dpQ$4$_itKROy%qI=qSxd^vYot-OSwkh%GQub-OEc~^`Q|*-laAqBgUqkUc z;z_M9Vopr`Y^Q_okr>m_qXEs*CyW=4n|{d}2J-D88vClMIb6Q?Y zXw~1tY9;J@!1<7IjXZn_oxS zdOPv1uk+t`tcjlYyHz$qnd&d+9Tyj$QF76U8BSCeaHE1Iv&mM(K6H`QUpd#won@WQ z5s|#p=eF5Ax!WhYH@G1{(WtBLjXH~oLYs=X$bDsF4mBD?hI78B`MOF&#~sK~{@Zyc zi1KbvN=_FAf$iuDcH_moC&}D-9ZMu$;q`L#+~xsxNQ1Z7=~eHg)`t^H3Geo-*0|o3 zx-DKL-_iH@4)>x79@NU0Ktbf6@B8O$co^y>e?FBO7E@_H%JgBx*_e^Hd!iljdHxy4n#DjWWbo*Macr=J@Cytx$ z=jEssOHvN3OwW_I{KckW>2CJR{r+&q1CBCKf5SZ z^pL@pxBIf*-?NP@v#R$2 zsfeDcj93uA1|Z`lfSt@C=>Bvda{;)sVcx7kazet)NGYqN_QNaSRdUjQRp}LxhPF*j zO#!=O5`r7}GM#^(svcOa9e4O+3~&8%y`p{zRQ+QNOiav+TE1%pVs6ez0*Tz%uqvMb z`bEQ0)QxXo>-%TqNwmhi%8K&xk9f=zg-k zfGyP26mwl4*k9^+@Q@szgv1R*CRbM<)QFEah>~*JIy&NGVqQfpt7`)+c5DZnFMwn} ze!Ls_A^GXkrw4d6>?|ypI5^b8uFU<1?r<@FeytK}sNVe!C%_yuHa7aB;|%3~`EYZ- z22%T!a{~D#RBuxV23$bq;0I^tvNUB7yQ^zxEG;hLQVBY)4`cv*nk=;jDZ+8Q7-6XN4NK-$)!C{i?#k(09+IPAC8_Py!n13jhu4Fg)m z`YiHMgO($dQv(?<@u&p%_B&{&Y^$d1;T(*N(j{f>8Yt0zxqmMB8=IJ+pFe-DuUo1S z3)oNBSoEj$rHXX4w$NV@iGBWYG4lT+_{sKvtg7H6chyL zdj`3PhZMZbf`Vs28fj~51A`8{DF-X7urH9_`cjkU?D>?j!AOaDSFm?q{CupCKWh$z zpgPx$GSB_R>&w$aQCbkO!99@T&zxTzMwls zIiP}OE-6Qdr7#@W#2lY9d%y7I=943H=ojRvuWO0n9}84iJjR>tC9Zr$d{1*e-(@DD z&LcABAv&!@lS0m`c3%(G5LRZ<_lG#xtm@=+=;d;*)jIxfXh>{A!b5n^5{$)GC;FDR zJ0q$8tZ9AuBfDbTS**N8$HJ+s)n#zKF?Z^XFO{IJGuOklc3EHVsW!Avi{dv~`yqaQ z@)PBAYDl+Kfb2%?pMOqG7xT0Y-IbuA>$SCUH(zCpf;&37wakcZ zH(gg139nG3rTn;__1=o(2_&WBz~HklqCVS{xY>*@Ka1^sN;||Gi?6`j``+W{><0;t zKOMHjL2S9^qGAnCK0OWCkNGRj-m6dIcbFW#sN_T1|Cf_`V;~B364@!FxM~*nO&l>7 z{Z8Q<8&AL=pQm%yv->+-6KHL7&*^8qk@ovas|QgD3&&pz2(N<;?B$0SWK_9SjjJ)C z4h>I^{<7Ua{jk?xvpM6^G+{&A8Iy;JH;v%1JPh&5_3+|6PJF&}{n3r5k>0xXWO}nB z-LE_FP04i1Y>CI{MNu3^hN!?Cft*YD_ZccZF6h!}{J_@dgr1-rE7WajU=0&rZb_XlPo%ph#=68a~;HH6r zf#|2I>I!DMoE#jZ7E2pK$3QHb_i6IwPSh=|38Ra95 z0S^Q4CkzH);NGN7i5TEu?CtH{-QA_7@6kz1m`z_1m0Wx?TzXx=C=XadU=@RcgMqpv zuc$cupyG&dKvftA z?Ua<1PqIHjjdOq;pVDY@gq0%_mAeQQVKsHUpgUc=dU<&P6AQ>u5WUs!w^JgD-sUogh>sL;jhQ|-v#@w@|NhGAD#$qR@9$UD90d&r zQ4X!3<6}!orWWxmAw{-w(3)~){a}2P?F7(1=vu3Z3P-(z{f+<1wUjXqEsD692 zAb!GjjOgks4kbu@+!)SHxN zUB2-mXW5DrgCXphtXHiwDrC-C$~fyDJ<4M}$ckdnqSKm!PH8Zx(Xh4{7BznM2~Rm_ z-fVTTL~`0F&GW;DltShLzq0|l}wtpn@CRKVYfF*8ExYtuKm+Jv#R@_?lcGBN0?nqZW=kWO;Q!?ryj$3V+=CaJD#dCtd zmod^(L63uAg#-~qf+}-*nwn4K8UKv4ye>2j5D{64B+T(2-lC36{J;hQY5ife5{&>BD#o!79xjfTnm|8%xD8_|DW|d($*w)r)QGPO zz=?`9G9bc`8GRt2qAK)=ykJhfQJg!&i&)xiLfxFs`t-N9G5{g~Bqa5!mWFWyb!KGb z%{2qiu)~6aUMVWFF*C0N0>IU^!Z$eVF$UPtv1VQX$N@zf>>{w2!nm^r$9qEKT+#jC z|NeN#9iFNV?1lGgPa+@-0M<-f1E6PjVc~tH9?)`bVJJTSGk{6Kww05WZU6IUPZZ=f zmh{_N`Nq0!jfHb-X=;)X5)M_`Xg3f79v0~9>swpNNlC2T&bGFNV5`GmKp+KV2e3K- zF9X9veDD48-|un}RPV!ZXhg?5I&uLvJ-71bUR_!uqM!iFtj%fq*ZWq$tNX6p00gI+ zDojF3N=i+gnx5|PJdu@+?eY{!OGPF6?)OKKl?Gg|r?)o<^bOLx2Xj?v+s6~QO~8Q} zU~k0>n!T1%iHhL#h2moXrfTcz-V!~4$)=@E2f2-#9cTc5gOtvu#~8qd07MzZ2w?W* z`T6k{f=fceuPX*bG&3I#dqXn+{A1*5UFC$LJ9&Y{YFLhriaL5!4Y+skC}79|`wKV+ zZOvQM4)S*aUmr76QzIy&#!jjTNS9-R2|xq^3=6O=w6W`lq<@dO3m~?@lR7#&Z{EZx zvS{H4jEs!5mmQy;p6=~2v$4T}k|r1dCN&74%@1e4+Kl&)j+9=#TAyq314<{*2af$U zp}A60QlI_(fehN8A~f9OhXM-g?MuG8-X=nthCGRR{>8HFrTE+NH-bL+zcE6?o|8Hh zRp4Uw%J-AC_TFU(uxwWerpvychKAn0;&;B9Q(y!an=SG6iEB{wv)idIr+$f)&xqKh zpnU&l<8&t9%&9)g@0YX!mBca(Y9ywyz`Y;Ym>S}t@4P1}zP>Mc^N7b#wds7uQx)DC zn=X^XiW}HU#|C2;u}lwa?`ib8QkLG}_nqZgoIJizCgz2*(rJ*4nf0Vj;Ps!4w-+4e zNBX7_%WhlqV80ft-_hy9L7aplJJ^*q1#5RV7^5xvrRx^Cxb4!M+bSZTY!02NO5V_2?`=J&asxDLk}W;?~j) zRF;3gHKhE0o}R9i`vCQ1B9@&^pfLTkZ}Us$@`A@y^l(5A2P`LQyeVwz*suSpw>SXL z^VCVB*=c9|GnbamR> zcx?I*Ekt$tSkfg8>i+2kRii(xS1Fl1YGa_uoLAJ<->z5SkeIuE?Z>O+mm^r(eFk<9 zS1_{lou$?y8$HVlBqZ67K-oQOQdnv&c?)=^lB#& zZQx>f+~Zdx1u0B#N4rN<%`CaKs=j_Ojb>n(Apt;=#NDd5{}s zhtqAmekcYdHvxbT7%fEqb0d!#w~Kzwr9aBb!&g^VY2KSGVnpll8=^-8FT=4J1C)~a zi}oL+agPDpab0p7BugA`m-k1ZX0^05M}g)im9Bd$7P?TaISSH1kJJ?M0dxem^tdhU zwEZO+IBfzB$2H=lig=85NM~hc9vSSNo+9Re$;yGjZoI+nUK$dQtt*^4L zI&QeX3D`Ao3B*;oupF-sz8vO-S{P8D9qsK|vI{E@gS6J8tmxVQKz9ry2{jndzj{ce1K!4NPyZWn{YoqdZqi z!tF)g@C2MV0w-gfr&m7s#!=eq7^tZ^%}ym@=$*q*RJ&fMd#}%%$iBW#D-fx0PQBg? zhALrIF54NFZGJQkqC(9pFBtD6J#t8P@U^fsJ@3*zjLgd^uy3$=_w@Ucjc;O;VD%Wm zh{^b$X3VOj&#@@oMWOBBE%=@{uXuE2!8BKfMP7e?kAd5Qd~j;a@f<2aBTMEXHJQ2( zlh^Bh|E59=B{)ylRMbs((q6?|DKVIP$uxN?KljMmQIsEtsLY2F=yEgTAXB=-wBj&t zXq$82M<#|8(|JvJA%T>1XP#hTO7k!?K1|r!u8a{YB=cDd=G|OD)8_T&Z8xO@-ryK~ z*!ZUH8l|a8SnBCz4<>zX7ZyXnvldE56|;afK?RIx>az9(%V&-O_&MX-u*dRu zRu!Z)KYlG0WUTG!|APfr#;mwdpZAqT2cy|UpJ&$JZTByjQIhk_bg-G?WvXfBdpHW~ z+u1N&-LZTAYCi(-GX=LPR%F0WEFp!kFrtTV%88nP7r4W_gdFO7UWFnRl~{`M^MxGt zNEovumoe#*7Jp5eNeN|j{?TaR9ks4hq>0Mv<4jVb@3&GKrgDc%g_-lHn#tspwK65( zC^V!}gvlD(SpF(MH287&AXS{td)sJGUlFPv>9tiim$j-9LeBgZcVxvglV9H zkUTNVlO)*R5Z`_VHT;CuVl4Dv2I*nq;??=f?uqi#6Cl^N>5_PVD!5Q_O!-X^9z-9z z^PB*O-sNu+b4Uz+X1=>FDPxwn#;elwYv0$buL#}7%?zLK#o*`U9b2pxi}G#6H3wMY z25ze(RvZNx<((UcmF3KU+?#As>4_PV3GxWcPqrsY>JJp&CKIS>YPxa5SBL8K_};nw z%*tw?cdG?*FWlRo--C}3C_0M6k14Z$`qKOeA8BL`;RqV>ewwA4U^UDVhu?LWlEW|0 zP1R2X{T8L3ws!2(_zWZ@U0lX8tc-2Y-; zJw=)B=X9>YeoKcM_~qz7aAA3yONpoa1MMY))kM5v+zQ)*d1(NCbf+|i)kM+XsYZz! z0k4AfJhSnAaH>gu$V|8L^QBwWQRsFtt4pf}9=QoP>`Gn6`N$;xz;QZz&8BrlET*1a z4i&~N`TIPY)LaCd83l=fFXW`S{cq3lLnf*9XKs7+?{n<`dHa9LRJ21&L%RnV$N*Aj z!6*#I8gdshNs+cDCMr68Or>xt3J>K5=gJ4H?!I{Yl7-}6^SfvIbe9Fp3U+oJo7Agq zTEhP$7XUKJ9toB$nCV+pCdsC*{M1*d^vz zCFRsUX{&p0Solw3=Eo`}Fi6X$-`}c~fHNGSXM?}Wd^j_L=c+P`pp1Dek(mJo=hK93VCdD_}AYK`{XiFAy$!WU1Btq_j zxq>+ARwS2>S5#C0HhK+Y9<8j%$jPNss;3g|)H)0V#$(WdxFpZ*Bldu*)_n3^J1nNU z-y}m6gCip5A$C{Fy*)jm0zb{o%{S#wBioHBjR3nsW=%bRH%BS)v((4Kn=)e+GDveu zan(=IGE1fOzSiP!OijFeAgVuAX)Rvk4~>Z-?qLQEAz5{;WJy@?r+8ohe-s)XaeMW3 zcZ&$L@iUEpGc(HI05QnD12Rl)Nreu(cZc`x3qaJ(f`0PbPv@Zoo;)!F!vP|uV*9j4 zD(GkcKIk&kr;nO{Ou!QeE#c5>5D@Odk&TD$yFkhwsb&Q#I~SJ&NcFqe1E6woQ%L;;Ki*%<^%d*u>!9U)`Nq&s-ZQDEs%yAjRfT-Q0=d}nj?pVy z$}7|19`Ogb8Us1+RLFj3e-bjF=4sn}#0ymT>5Q@VfX%I~f~PY*JxXgvy%L)* zK*raECu!k+D(FGVbF&b|wR&@KX3=F;N$g1QX&b`O$L{*#xR0$;haIGa**#CoC~*bJ zZptlyVVtW6Y!{eiFqEU`44|W_fDfy2T4E>dM#B@G*Jr=@*16ag+K-b1=!R$}Wou`~ zU$zrvwJvMEAcL*M&>g@TPMA!#X%gC;^M}B?t-EqOHNIZbLYR&@Yr?Bmg`??Q3}|P^#9qI%J-8t z0$C@MAlX8|ZgND>IS1JS#H`yXF(m_o$cNC+vXF%4>EJnD$T6v;)BpeaDN|A zxZVBz2*u;b_VE+s912t}NTz7Azj^7iz}=^%H|LFnvhL)Oo2AoC$E9bCJMW|Xi z?UMgmE;Yr$x7%w?w6c(o--`Ugv9irL5p3j=5{GDpkoFaU2>4s}xb{yAp(RcNzPUf_ zaOrt{X_KkSh59#!a&b~F_fZzenN~`8mk+AataGWccln6Oj=rb?_Ugzn5 zA9#r^S2Z<&P1a^lso)iox#zEx@qqeRifBky?Q$yvQ*4QV*jrx21Yo9n3~CskdTI!t zFw&WRb7mkjm+c$pSxP=RUrP3Znn3AL%K$tjo^=-@M`FebC$)){-(`QmZ=C(o86pChg5CO-+{z8iJb z++mCmVBt8&_o`>yDV?hmei^IlTu|Cq-D?K_V~cQVpXqb*C)7SSAS>iZjxbo7GrpDb z2~Fgd_rs@D54C10S`Z|8Kc0j-3Z^JA@gzQEZTunPE;;Y3q4!3wE3VgEF`Mw}OHo3a zof1rj+GZwzWR9>$YbLWT-P^}!A!#UAmDJ2Ihd&}LY<-91AvjpE{rfk>P$-l9FWIi2 z+?MHirj2$wz0Cx5H=n@)2Ex@zIesyznD3(nl!PoC7D-mepk+Uh`u^Z0I)=2+;QMVc z-PNGp#`e(;25wx^3ba3g3kms07SK2*!5l8JL$KHE+NR~)*gu*3sOtP0^gZXjbo`Zg&!=*_6C_GoL}Q^j$9JhprLGOLNobAe|mb=!s>NDYvXPH zyDg5{c5xY_ZLA>#;=yTh%|mYbb&6oFhpe$7GEK)NL{!vhaS^5SY(_&?W<~<1$=AM3 z#8WH(7j17D7G)Rpj~Ym~Gy^Eo3|#{P0#Z^^BHi6BlF~6UN~a(oB{_6ROM`TGHwe-a zXXEp}|LdId^<0M!Fbp%p%zf{*_WH$I_nHyXaF+x1RlgIRYH_;8*L!wW3>TwqNIKAS zxVbbeT@a(rI{Af*ZTs82BcY2VrB*6$JcAB`pP!XYlyQO$gV$a(9S=2-6&d4xcX&uO zpm*nsbi$Rncq@~`9b@uYc^#z~jkQ$xp8dUx5xeR+{>2~#t=XbFDId4`#r`9$Nz8Cu zxBM~dhl(GbE^lJ%LzitGo%E|LM=H4TiG3<}R&o-*mwm_RXL)hF4V9rxX-8SB3wtDL z{)6G&!F;56QDo0}v8dEIcR^JS){3zH)jqT)NB+QO-==QxrPC*>zsc-QS+L?QRg1>7 z(Z6BfSM={}4Z}W9sJ_i<)CF}vmaLMF{eOR_UKE&SuhvzSG079Xw* zAjospiXo?acJ=%$xut>o-mSlr<+5p`x|*BC>Pbv08c0n05zV?I+5Z!Ez; z9@e!RPBlCdV)K8Pu;)*ry9+@_;h&f9^0|I^)}ocH-mV;TUwwY|Tuqod;gcp+@T|0* ztLp|GVYr7d;!~m7uqsQOF_KJF)e6qUZhB_a3a$2%9=v#qPVDFb73Vx)9u- zr+wb4&O&U()0tyuhPr^c=3IM%JG;9c=dQ7&=IK3BiW!45M_!(q46;(3Z>5QMki1y3 zreD+y+F;zEv`=2yYW83MQhPkE5(S14cGjxM$|pAXK8gwUw3_3tM(Yd+Avpi`&g)XT z<^b<#vndLo@D=IV)J+o)zLPWGpUjs6?buOw27BJ7U8I9b# zU~JHWVu-KvgUy;@J#)TySIutQn6^oLs zfLx6i?KxB60Tkn#kGimI>PnMP$6Z?jKk!Z2c^^2m;^B>Fb7<%<^&9iTR9-ElYtgLZ zL-Hqfva+&zf#T0X#cYbytoYmT^QTtrE08KRKO%?KxSTp*`=YFS!*X2lpJPRuzn%S# z`%NypUzSn*QSQ9Rq246_#JPfC4A~@GdY&(JsBOMoFi6^U8EjJ*|3br2aY9Zu&Zdsv zE?wJCqi~l8uUxE}Z1>l_`t*BU47cn$L9l5w_lCYT=OSCafVHti|EYn~&m1)~***Oc z+9eY_Ngr3-XW2BSxJQL361wD>%@$y&{rUpDVmc z($`_&JYw`I&sej`TC;rkWTt6{qt`rCtIM*$LAD}@#K8ol@CD<=CD zgelI72^JzG6oKYESZ}Nry7?)^%+h1kS%P1d9-n!NrhjTT?yu5C{-{0FkQS;-FHvT~ z#>U><_fY|bwB0q7yf99y)(PiP^L)_hh|TXI&m9NuD>7>8j(Gn|=SN!22vaF&wyKq* z;~*{0J8%@qDkmd#lZpG05*o3^&i&)qyp&=IS&>N!(hKX-r$j%K;+;hZ!(r4~K`DSs z2#DC2U$z4)AaZ0mnLRtWSN#v4UhlExq6}+q&@rDLLZGlUnY~i`zQO8PF`wR^-1n0X z?PMi(Kg9{coY#m69o><*eseQ;wb?JU>9JhP_Tu@Uy@&63Mp{HUjr_!!v>0N*?hd3W zM7uHYXa4-#C>3|~_j-Av&y@S_UP;<0SyNDYp2ip+e^~|H9qwQt3cMYNLXmHuyLw8k z#VarvO)Wx6n45u}2n<2P$8qo(ZJs4~)m`<~Ik51gd*cs&FKBFX;{L7*nj)%hsx)m? zFMT6F^CVZQvt*BKX^5g|{VLECEFyPQXe0p*-qvKAB6T=lqEw^7|;E;*G3!3|&QCJ^rJ)tgRh8fznva+&VG91Vdc2f;Y z4Q^XPBDt=!(6;stJ!Z>L>7!&fQ|ROo;gXqJu}ux8<{?ei*CwJ0{$Nj7S zlhwcj15cy+f#9r;^z_ zO%tke3X;^a*nF(#j1qHd`{)=L8A12-L<#!#iD$MdLAv-8d`)d-3?%kc*E1r_+K&Xr zufS(gW~#Wv(&||&E7aC0t>`5x*)@0;nqR1->gZ+2rd@uGD-gmTAyf;0Bts>)%$n-~ zPLkbE-hN=t{_EX{<6x$wc!!D$ElDo5^DTL|J?PfKV$oyxeBvoW0`JaHzBCqfo$(fR z5PXb z(btbQ3W%`fyh5g6bk#}LC#pD$77_yqwqI-_A|hYDe5o)7b{&n~@7lk+h*q~8pPau~=sQaKO z#R>Q?-7Rv3^-Ctu(9p8o!EWFsyp#?MA}1dv38RPkBpJUbk;0{RV@C8@H5|0yZ`e-L z;dShalri1k3V)8^6%A&Yx)%sQaL-Op**(e|^7h#pgS2h2S;w=`p9X!7UrH{fZ{yQr zrPI7ifNqMUKFFZYk$65x8+X&;a8_?r!`$AmGOTmnTGeHux4FxpmiR7qP#9SJ6qt z59aJg)-%P~l=xO@O0ZCaf2EC-f8Q%j8zqrZYAx7djPw?5y5#G>tA2Bm2=K!t?R_(37 zwJRYbTduqma~_dYcd)>gldN!J=fc}}O;VdZp?B>lS@r4JWpRaAtSZQ(QL<2jwah!T zqCZ4fT)F-%1%lcsJ2Mm8!ljMeAoKWUjMQXLh+(bOWsj4SqK)!wNb6m54D^U|uGP&@ zCL<#YYc~cFt%ViicGr1wZeLqc}jSrdA}T%RaeNpG+CI-PGpu7mVFYur$N^`FZhV!hc;+!A#1 z)g|6bK8ce9|LdD7Gid@f1+cQT;9_Gxu{oE)3J#!u4!@#jhCkHbQ)tTnm=Zvua8 z)-9j?+`IPkWABcw}{4kRg35NKNXc@^hsA9+qAdJ1T;>v&A9>{zAQSxK3 zM2o7r2uSp1Dr~W(aqA-|lKaRoq6mL?Cv~qf-@SXNon#Q59850QxF-fV#HF*I8cUBQ z4xBgcq@T}c4$c?}!@;p(B8u)d0&_m-CEwaw-cohuCu4RZOQr3N!NIjOW( zSBZm?0sHgAGU=D<1#e8z1c@*o!LJU#W*33eW?e}^fy+0xgY}T$Jq|7|;?pM`d;CXY zn z|AmXAONL#XH`+`JY?w_jLA%|Ytt{oT7~lwfml|=b_0xz)WhC( zS-NOhy5IHMMa$s$fb|c?tV8@v$PA*vDeush3 zmk>cd7z{mTh}KUyuiEViJ`)ip{#ON&?8Y(cH@7w!%Ha2Y6l0ocI>p7azg+M?ocZ6?y;Bg>Z#Loiuv_7HS}%(b5wWKcs=wG(WFT z=1wv}fGXB^DJ>&&7=|s`MX@4QtJ?ul{SuCqA-|lSt;!rF`*YCiWu6YQvBcEV%J2$Pui|PIg_`1ES z8SBbqbd*48mN-!D^k4EB+b71rLz&VC&>r~jlr)J_M7FX4pboOy@hqZ@v#L`QKT$L0Xb*_)(BO@!I=5o?qMU4?442N3N zf}PdP1>gBIYld=HG@my}7*wKeQ7`BWt#$w?eozreVC3O3X*|UtM-H7=?EQ}Il`@?o zO;|W)&90au(BRdgjQZZsG+>>a(tp`!U8FF)YniGLP1W>K{?_F{vYV*k|*nH$}JKZ__(1dCx5^R1U%kSsc76Npj-M21u!JoWPyn+!kVBGn${-dBcF~D zEI|uaHa_|yTAe3QFUkY7R>7z2l9Xo6f3^*8ikGARh+C&A`J~RKTxxA7k{_HYI|JQL zvW;O=^wsVSd#vKlgPk2fTuqubi|Ti-99mamqJ7HVN6Zyz{mBqM^nl>9$wY;;8Rg#) zUOv(?Xw-b)Z-Ti&25FN?s?%nF0V9c&2994S&9`C+M+KTF5=uz9H^w$Kq%Ab@T(YKp zQ8Qn?%=@-3vxQ6V24Od6?1=pd3Z4bOo8GkK*N$R%(ZWTrP8eW9D-CZcLhA&a3+RG$ z4lwyFC>rQaw1tX`D55kdypdWuKZn%`bP)Z8&?hNmVq_?s25Z0{c;ZJ&gUraD@UoAp zgI<5;oCTuq-_DQ*VBJ-4^u_fJKpdbw?;Hcl&$K?Jit3|kNkfI-DwbTgOlyeaP!QxW znk0+VPkbcq%1i>!5KbHfn<`*`oti;120L3XgaRk1EHX0EtXxZ&Hq*Ug5~LX^@JYV( zn}Zf=WSmm&$z{%n)rC!iekjT@CE#8eVxqoA*9W2%eVV{X?w-#%6<6zkYKg8*n&vGI z4t^djK3OmA#m@)n%Qai<%>9X^sG{oOmQN88)C}4R=4!XF6mu>tQhvOsi^4Jrh41p3$ql^t=LZ zeK!0fC=7(DJGuRmw2j__bWSJ}t54SH%~l!X5m_8tW9dg?B3Yyn^Ig#X3B{+*tRW@{ z1}!k#rt}+s8|ls3*wOdZv)|kZTU;zzas=jLNE^xcs1AeQu$b{icbwRhdo9~j-469v zR5RM1mEUqENb|+HGnCOkY1)H`iIaOoATn#{@Lc3w!wJ=ly2G;{=1;ib&eJK_53Uejj5ANI88VZ%r%4oOq>KdQp0XdxS*SCA9+k(`ylmNLo6%K!>)kGN`Rf zoSD+TO$+oeR68dwOYQsj4NyAqpG!O7pgbrO7PI{V!b!8}<&#G|&DNC>zLAIz=y>Y` z#`3{!Mz(!uPixRR7ms2&BA^1zRgT_}xuUqb#U0i{+Cizv)7Aq8h8S5o*NGmQkOd|G>0W7P=O7A>s>WRiE^m zf0kz|%DUQnN_aN9>sM*P$R&BERhAYzyUQ-^#deIg_o)0^&Vp@iSc%y02OxWbmH^Gt z@RaQJd2a;UzGWk{vr}vwij4>l;Q>iYleNEPaT4?Yj&=oBWxs$uE^aOeN;4H_z7|VT ze(E8tG)Am2GTZ1=Kd>?Lo9_!S?7BU0F46_n zPC4y9<^OsCj*6m9Q7kg3A%P3>to#nhR(S3zD`*Ae9TmQWx3}g0@VoVuv3a#CoF7d6t1N+}31o4r9rOHGK2qG#^=qimv(Z44 z{?_NRat_z##_ATLU2T%`c!(`L?PLYj$A{YjJ11kT0UuZYaCkk2-^(^6$#!!Pm3J&m zlFg1O4qnuJ{ra_q1!EByPM$$r|6OCZ_JIbp+Rp%YQr$mg0i)EI^!PhFj+ditQd)52 zsP|r(af7-sfNnPFlC;RDc!%1GyS={rf~*v!pzsByv!gedo3yskzFz*)0Q%H) zA=UI6{G_lxnWM(coJad2vox~MT%T2)PDZ8PHs3|bdT+i^ems6`bmq;t0(fy#lhR*> znh^Io-Vw>$Ja=M%d*WT@SHF(B1UJaSi7R#cHvS+r`;Nj^q~4XjQe=G>-)q*X_(Lz* zO>nx5iNsdxEV{7F{0@9p7*CKpqt4YnU}X8y_5hT=<8OmaykgDMxummHIu)%G3~wtv zWXTW(`YGSte(A)2cG=X@oDRI((W>}Q;_vq5z1%y%#nnq5`1=a{lp)ot)TW}KXg%~9 zoRG_Y5cujWb?wjlNg7bv8yeoTs98!JQD=F>L=?Plw+1qI!NtC4fbM#oI40Ys(xRk{Y42Yx zsN7tpZP21^UDPd9=(1-u;(M{gMYEXDYz+;m^pfY|i>kKq|Ej^6NTDjwWQqHBb%~xW zkV%9QO^RnJnm(}R%tR^5U^k994JofKte>BDnLx6}W%B_FwIa$#?yEf6blwK&-`Pbo zT=peu)RNC2W2!3l3Sxi;n%F^yjive*HF-AB15Lo;91n5>2*&F)!a80qTY=&6r%vQy zrIgvuiBOfX5NgQ=4@Z5EKBEqR4_d4uv9t9+_t@A;MW8VG0M%l^m8 z+!8lQ?E>y_Myu^(hp7AH3xfq6m9qGS8k}2)b?S~N9?K|Tzpq8`SY7$PV9(5{DOY9+ zvtRrmp8CebfkK3|*MN;}r??tXyHkQ+uewzv$jgf&0&pFu9Z9IlWnpo0TFEu~CB2PP zOoYEOTBP>@xu7L)G<$GxF!7w|X|I0^7iuTJb#L^TDb;R8FwXFx)Y21mIiP7S4-)AT zxu{XASh#_=y>p3Kra!BxXG=wiA7dXUThYAE|%nMsp7uB_#EyPBi< z3LDc*Mq5qj&__wv$jWMU{*?C9b=b$KmmB7&F&6M&y`x=DzweOCCKFDtAE!hnCiqK z2T)!e22r9T%vRk3Ev{|-OfSr<{)1CuF96lkB|?S`KBuH0w7s5Xb#3^R?{q{WrkF$a%Tb)G42mUhKZ)Q=B0Gns4dOo(`Cja4m$Fnl0I>JXGI+*QDBygCG zNictXHu21__i;5frt1;wM1MPsxoKSyCY##tuos`e^cauZ1l=s2B1}l)!1>*D0CXU! zo^A+tl$3Itv)(|vuR08L2p$c(gVSJu{cf?!_r17AViUmqO(OwKu$G zUS%>Hp`4gk)iivTyIvTaWzZ#fUpxdX!>vqvnoa2xdnU0n!zNJd{zt;}?SKfrQl3sa z0AmYZ@>1-9;4tC#SonlM;Nsx{plQk-uK$T9G5TpdH^BVZs05Img98o{IHHAXmA-`w zg?^Fm!<*2E=0Oz;Tq3Cp0!Uzf1ThRWr>Wq|%Di~{su1!sVHLH@-P>E28DM)d%wSGI zQj8~N^;b*eah4ehW^8_MECNjOh}OGd4ZOV0i{KycK;2c zy6M+a(QLItdV7q8%r2_OHvr*5Nzb&gUi+LSj^3tep~+!E zw6>AyIbm?iX7~`fpau)^H?@5AWIJYn$3bYJIzh2=xT>v;?hxre?(KPDpc`_@k*twM z;kj!yO`RuD0U?(DNtpTru&aJ!-jHAlf5c>l@0Z$CB z?=m2CGAIqWG(FyXt zkPv2r^4&~R5yww9wpD>|UTxG!QCHR%XjF8@9fckDajTZ%yKl*=ZR%cIPfM@IS-+ps zpp!EyGse8~zS@`^4|C_Uq3CSV)*hF#-@jW2NDqu=tUAK{A-y_sc*R%V9Cj0Xw?TeEPNaMq!5Cay9$ew^7QBee& z=_?j#+~~_X_vvWA^NWfiKF4fc1Du+gJY=a}ENdvSiy;yP2DPf@6QQG+)}g*FF6^pc z@_|99X=oCDadLCh4xyL=dN1TfM$qYOoGPf+#k%R8>LGMgSk8J);=b^o--K0B>#vZ! zGOEo-?l6iIIG{Cugm`|F_I;)n^ftM&Mxh_fDZuwFn`M3_m8Ak5CViTPsVdz2Wq3SZ z>iRUt>9!!-Z0=;7{+&$rFeNKRBw$aU*?*9-MPyp$$jjO`BpL~ z`xRi24s+%fLuEYfC$mduf2jgZYITcM^B3#r(GkJ-w-p?*3$B0rW05?n0x+NYS+?q{ zMu^1c2@zOG+de%RTh&28Qfp!}icWwDyR&BO0V=x?_J{Zak{56nDi|6MarcI!D;v&PYD$rQObXf$*~BUPD*03T@<87W#)p1Yz+S~g|qs8B-s9}VPg z&!sS;YfcE7;=q_V$n0ARpK2nB>43tpU>Bcwf=waX#%KV=dSRA`&cA^^$tBw z7`NicjKqr9Pzp78O2q(PjRgL#;jQ7C1_Lna`Gi_r5H&S~rDD1tWJD zYkaS}wyPoYKE8ypAkcCpAbSk2-r$*#!Z_~HR>k$sk(V>{I09u3iB&r5_q@;GD*zT#Kr67Gc#uIO6@WD zGK@^sklFDF0~Mput>eny@P}J$z;ifL)P*y4WFgRWd~?)hN5Pv z<|UZK47JW+bAsgK5oU+s6-x!FD=;)s$x zJ~=VtCXcF$Emh6e99;>av~T8Lf&J)V2lanMgvn>AvB7amptF8pb+phvmP`P$9MRHC zU1#>Hg@exGt<&#Q!Q{B1Dr%a=j)6vh?e(g;C?r1Al!FkFnG4h0&93APS~vinRq%xq z+6H@z+8QLL%n}RqEcZ1!%0$39NIbHC3x&( z^Xrn*F;10^c^cV4?#nJgU*~;VxvsMW1-N`3E zj8$2L@@FvRm^a_E#I=%tKhMo$ils{*FfjlM@mnYfa_`65=K&4R;vfRfvFg~zl)G6w zV>>4b41qlI0EinEY_)JDEww?#`1ag>o(DGUyX~lL?^wFjFzOd+IDI8rPyHW!wH&6A z@mQso@+(yqb}d~%^f86^6{u+Gy>=`n!@h^EQ9(qFzdH#a zCdjrtZPT=U|8(Eqzu4mBexC!e54>xCqQ-~eW=72=>MXz`43s3SU0O_WoLm@G5 z19Y(xY_Lab_u9-2n^GPuemhx$N46V-SR#W#BBQMOopT?u!ZwH*&8Pt@hn#l5O!aYKa=I9_>!1dT5@U0VBwYIgkCS?B3vy3 zF(PG9VlGr?Ars=dT1o>9yjLTT;EDY!hF5koaZv-8Sw`GK>dAq{;6KF(wR01$hk`GJ zkc|pK>R5S%FMrW47~3Uz2*qcnEyxasbekHlZOxLaBfNZ>*OD> zi~v0*p3nxn;jO~nBb;OniaKJ6jL)X4iCNdQxR-6T+UIPuxhR&p>NxC6hMX`k*a}#4 z+m{CimEnw+VN{>Ed%Hior#I*&F1NJnJT4h9@>vp>F+ z$hhYgC5|F*cMJ!eU=)$de{C8BPZ9%|#JQ6s4I(6I!NU^!4Lw7THqUZ_NFp~U+G~J& zeSMub{rh*RdOxypWBlb~%wSNQV6SzXmT447{VH7~`q3UzH4x60Db4e$Al&O2fT%h+ zCMbCb~`R-Rq)POi|V56l=iyBn=L z3qoZ;mLof6qp*3$30N9sCX_YcSge~p9v0#Uo)Iys6VsS(QvlrDA3uI9{Mm))N%1AP zLYg@ndkgKA+AWKGwX0k1*7^B)CP{l08RWU+BA5(;2Cii~otKJ^5Pxu_tyS7sfE{2$ zrhuO&Ei8jlwsWK67`(k(9kQl?Ytb&!-v9JZs#IPEA5@C8UFy}~f-)I2vU;=?NhhoI z&Skp5ygFv~arNz>pFDC+rX0l$3`E^NsdH7gg29;CnBii_aCV#`(${J2BBuy2$}OZs z$Z&6qbT2$BX;H1>An5yBU0?aJ2mben9>Q0ZJ+zdcdS==@7;cHB%6PTPM9PIzO4P=# zP#7Cc(z0kap($?yK67lZnfI%QXXUOq4Hc&$=>NQgOhpR^q)D3%-Nn=?7rj&ci?Mo~ zu#j`HO6}mv(BIWUk0IrKWNa;57mLy%;@(B{_kDmMaHc5dsPG`4_33?-&cPC@9WUG{ zsm}nUt{1?CsvMCmT-Z=&dI)as%IRXF>UT}3cHeVvy3p4-+IWbk)PNVp_Y3W0!AI}1 z56NUii0Z8oLE0Rrp|BjRk%O~mFp^K50G^sTv2?tbq5nxpLG~2kb*wG7Lk1af_>iZP z*#8)!Lz7Fr>7a<=HyQXK{@04LO2~^L%7np>E4!Usbl(z2d{ab9%quOCWc@wka?AUE zV{9sCD0@7k=9%sUVeCofICdA2GKSrP@n~C`kY;x6)gym9p8L}?)5XMy=uw9%^&XMY zJknwNbs_6uJs9Y15y0J5b+bj6|GWKKOR{?M!!s=d$t)E~LPHT=9`3TKLKK&iMY82m zXO#ex#H*bfCGWXRRz?XNz`g31gi1WcKs;4cus)WuFXEjt)rVC$OAfg<&v{ z-(M}E+rGewBu@(}&>jaB6}C4dY9 z{%i>3NYb2aVpsBQ3L6I~5C#)iK)0?D>I|%?v=|@E4OLX-NEhlW5HkiEdXq5nx%E#CW(o5i?q@=m~Zn`(f zZcu1(9ntLb2SMCwX7EwU=cKkTch+X^bk9WZ6r!ahD8L9V&Jct@Ofp_ToHDS1#L2zV zqG>65|ha zNxcwSN3ynLUMKU~mws@I%eZl*BR^h-bz9%(fJ7Q$evB)}qLO0&9R z0GV(_P%U4ihgoZ?!a9S%@8#HFn9wOxIxNJZ2hfIaC|+@Q@7W_Iw&B;QfeYvpPTGq= zDe`>@Sm|0tfQdXrfZYa&L7iEjTmcUFB{tj}n5oTfoLdd!0DjA@JSnjBF-ii_S{ z&(X(X7Bmg`28bvnCg9ya{gUZ+t0kJwkOCiPQVV3OGAhB1oX>fg| zR8Fg?CMEU+c)Rg(JzGNWFbWW+^b>(YEWA4%DE$>VSppPVo1b7+N*T&Phm%DS=n@aZ zvX0HVk!AO+*DTZbRXLe~;%x^=Wrb}nMmvo6LVG^g&9lZ~PJemG6y@8pNCrgvw+<6- z#Qg6Aiys<5aiKsbO7vW=zUEKOAjAztS{zMIYI2f66M84l!CtrSTnjGr0!kCg(kV2A zgCaxB(G4{OqCiIo_D$IDHm7%FWN=5BEtxn&8`afdIOkk+H(lQvFy!R{X|CP<8W?8M ziYV=Co>D}fo}JB1C~8~#k2IOW*A*C^wIaiMhx|eoCdCLCXEl}3oC=fasUVvlJ)1D;2`jqgdeB2)euq0p((?fqT?xa`?ogpUpO*2a9aF3(Oz%*?n z2^>?3g#@0*@heqQ|1yxIB}bLH2~7Q1FaU*l216A}^)YunH^_CVuPAXtw~QGn@lZRv zE(2P)214>Nu$d=K0Y8On4fG<=y;rgUNLI*D_yo^Bn3nhI4rUVORxb_S`SX=>sXnEt z$z5Fg3ZA1d4lM~`8-ExfG7@{j*Pz}d@!~2UB+51;=5TtMd?{Xvv5g%jYTTW2&Eowh zaA2UzVAJIL`+@buT0~7aT7ltZ#^*KJmvmu59VSK9)`2i@J zbbJzux7wH3S0$PCbMJ-MhV$y{k0jM4$J1>8N0a&zApA%7Sbp<*3zbKjfh~9i`9_?Av2VmHV27AI9 zKVbI$Y_y|xYUgX0FDAOqj0c+JRhoT_Bg5@5LW#BiU!c252zgwNcSyr)1DI_dO+RMY zpWR3j&mCEB?wIboz4qWNq1Wur$RIs+A`@V;-0Rvmty+2)SKw$R-hy%|cAR|@m~>v_ zP`g@*%9wqq>?eL?L-Awu(Au}T4xzH3>Dmm)6X@xG1+E~gzE5$1jxi`#mb=^TpW z#GR8<3GKoC@ReA(Y#=S0zshKO>M>V$hUN6f5mxy6F_l?Zz>=Y5!&SfNb5O}kfErq- zS#dKB{=w}5EPId^^Y|5v=Nm0Js8O=JGu3z*wAVKvOnFm%o96M|RRFdhqT!X>DH@?g zTt3DPZLf%<+1KKz%|n-3qIx#31vktXw$vqv_2MXJdN_)@B|izrhqKTg!GFzkEGDiP z68~o)i;iuv00uGb=JvPB$HSgJ|ln>sI#~;g6#%)XHN$O zB`#`=-=CzmEher8(6CZKQW6tCyYV3c^wb?~#m&AF*Gm^of{J8~kKMo%q)$@JU|BkD zF#W<2F9)V`Xce0n!kVun(Ct_z#rPZ zxw#<;1_IM(6FNe1?~JL`dewYMS^|{aRa$BCxz2GAhGkRGZ+5Q+)8N78nLzNU)Mj%I z0nydF#=yY9S=mn6t_oP0pz2_3G{6!9zq#}&^2rw^d6GN*5F9%p#ATqBz!7m&okhcH zW)GYWNR2B_5Oga7XzF7Ualun=88cZr!Y=xlZbQi&;(9AB0Aoe(zUG%d?2KRV5s;Y?n_~U;tPHlm_|g+M}ZPKsO1PjY+1b;{>mBFc_2@ql2vvGr@&0!GZ%) z1q`5}$w{&hZcy%=*#IZ*LyOPoUO-@>p{eG;hzT1R*3OlJoOIH|))P;hBO(?jySf1S z2ksFGYm%1jk#Z8Ct9Uj}Y0|J_4sY?MzFy9aeVw%`10F&{L)zIW%+v&=4bIn?sDZ$L z>G573@&X7iahALQ=mRd#%sBT&SPj5Q;O|U?M-upoX>sTdPw+n{M$yr}2G&0SJuoh-4x=EqBtbVQ5N)lT?%JHWw88okPeh!2naU-~**M4l$L@I5Dl6gV zM`B$QqL?PkkG|PiJ(uL!>kozg)LWVty4-l@j!|P7tLIlLhFRNHW7X{`C>6JG9)#7; zSgnNu|K5`Oph=}%e?1TTKgZm1@O;%f4u!JQIZRH0z!t-u7NkZj9qNR#HOiV z9GRg<=`1xKCWOe-5ps4x-cI!d;d9~qt=8ISV2+3k&IRHCMiJo1oOGV_ENOOX1egF* zro99dp$$&;u5gJHN{#s38}UrQtIJq$KGcf&w*m(4lCW<7T* z!m@4_t1HUT2kCd{Nh@{P+eB#4QA)d7*flW_EksC^PKZ|$$aoJroqsA;KHRU7oAz zG6i(RlECgqTYk7Q#x6#p4}#%Yg#SYSQDtCByV;~GfIx+XSZMn*fFbiynJqAXn$q7B zdf^#xsteuBEUn1|H44GiI}w0~BRYl&%q zP++V2mDz_{e?u4Z3Wd-uYP#HBUPV4TdKH`_xZ%nnEWo7hD>#H%+;oH(v659gTBG(- zl=s?r`^^}*z||t2Kksev83}lVfrSclu1&oedZ_ixHrky_(FF_%7EO0(6W9ck7{L`l zg)1vef1d8PTlY33AteN8M7ad>LCx8ToiJ1V%H2X@l>H=HN!x> znh^v7r&KEmPEiFyZ}(qeyV-&?*W$>{JEHuhje(eY#Cj(0i2eA3c|%_&b|!z>kx_#O ziKnNh(DUb3cO6belVerJ-N}#A@u6ghP~zUTIGf1+5%zypIf{R7wn1avM>Jikx;8jf zy~Y_Fpj+zj?z_Y|?%q+mCzBj_5KxWwzyAN;H)Tbt9|3x{WOqEU>t1MQ8u2rs{MV%= zct+Nf$g_c2*&06exaXU&u0b)?eD#k_HqsMaNnnfXw`-@cmJjXZ#IA~eG{S4KPRX!9 zAN2W+Q5DzU?juH;7bi6XJ3K_4rUL5}F1I(6O$*NLZ1r{SNxT;rQ>hw-Mr3DKi(3tT zI!VHU{ZIcuA8X_n={3HnV}N*_z%9E8B{cjUUi{sg?$&;7fhPr0Jk^(;Tv4r?)jfyC z>($v;m_eTvE!rtU>JGYIi{4b5IZRZ25!ql?x*n)9q5Siw=@xJ8BWA-MAL%av%P#P0 z@mm*G4rmh@m9|}%LOYB*)V@-RT@01RaZY}FiQBW=hptR&%&2C+I##wex2N0}UcSEl zG;uR39Q+bk8+VD#nF4z;(6z*0T1F-e@G7=#8b0Tvq8Fk1bL9#S-~X#|%{0dNs7(#5 z1K^>2x&zOj%5U2I3)j^CUCvf>;A*GifhMXK-a|H+C4&N zfbhSQprzyLRnOzlcO~(gbeDRnwnO*0edzhTCt9@{eGoL#t?^I#6pYWt{?N?>7Q-0y z*+sFmK_bf_5_v0`z_ z;WA+|k?$m_-8;UrstH|N?z>V59pbY1^&b1vzzR>Cpva5J$O@t7LR0%egAsbI9#hI3 zHRlJu7gw+6E*j7f1LbTOPgiH1^*7$rVESzmKQ}PaqBCS#`3^6^Y zzMNx4E#h-LkmYAjCS6_q_o2bx=F?w$A>8Zh8}rD&RB!b?%-U>)xevQGXs*25Y;`Zx z-Zon9z!9%$J{=(ZwtwgPZj~8aeqy{a`5i4@(&=@ec1A+Y9QM>E+5D}{+sg>o8~>}t z>_GibXQ#Jya~p>~u%o4{+)uvXnL-~CAybK|U#^Y^d@jLq8{v&Hc%8Q3<{LV6cr)?C z?<(ogpm8_h_D|C7RVU3&*1$^rg@uDC-@EOkn1NZ>^PKr#`urlmZAuEy_tq<^J^x`R z>IoQ%Ro&+#eI1Gp_SVDXJ(FV%HjB~zR}bIM9a`mfPW(Z8d($Fc)$q-TKUNiO!RdEh ziT*p|^`9k8R}(7@EEI^M!PZxb~-CHxFa_`}I{Gj)lV`0lo8_ z5tm#Xv#aS|zrF5!Mm!OpuATQciW{IGV8&_cukWRg|DHc?){0(2gUtZenJXC;sa+G( zMcpb}sQr}B{C>*3_sH(mE^cMp#^K!2x{ke|H{6IU;ez9D5|*F&e5&8+XgSB9Cg^ns zZv=rb$N5aj#a@%w=7C4Weehvt9M-|597AKqN0#QkkGa{~`+Dl5Qkm?;<3jP%l9B|x zJ7!%?^qCEEC%_c-jYH5U$&1Uo-`{kWyJR6`oeD7WT zV=eyOk7FvoKZ^@}{N^v-FEAeX9`74?U6eFzwF(5k3vN2j;^;59kiSas2G5dtEj+!p zx6LWkcscp@w(KZDZ@&1&`;l}>eX`oK&a((&ENbD?#d(L_$lZvm8%p2XfsWh4V~?$B zgTFH+qPJr><%g$>GwC8HyE3;cS)Dnb_NFQ=4%WO3+-?dQ{dWJ7x*Gfzz`N-u{p3Bg zbLhL>pI*1oc{c038L#G&%Rc9{#aMj)(?1kXK_DmpA&4`(^!ql+o$J-}m(Q1!U(19J zc%93)Q?hJ7(hE_wHut-NlOp|@7x%0f-;w~p6KUg>r3EGCpDH}RIXXEp(oc+nhl$TjAu_S;KYZg5a_I?eWKwwwlZXob>md z7oEB5v8p*%whQjv2Dv|)lA3O<#+DZ5#{?=ia_cF1MsgcZ`etq1kFS<@XdTki@(1#N zDIM@lG@cCU?DxrAx8RQ&le!IZbrp8bo8k03K4|oad^qzSbv;yj3}ZLDA;=2@?s@QT6PtZdC!|hO8d8vJPep zfIfV!c%iS)aWbQsHAU{L3rXzOB%Hy(TP3rtPFePaXEzOP3Vl&6*5ISK`(*e~o;##4rf*$#7qVCn?kZph={K%XdAW5!#`HOPWJY*GW}FPm?g z&(rw(OAcCe{Si#pd|us@L1mQ5Uk^+TYGr(h&R_(5h?F~YZi}Kq3R98eIQ>ilhlMqp zxURN7)4K@PXce8QnNcif{MJnLk6N68ok5SLPtO~)_7VoOay^L5n{q%8U%)e#-6i$4 z7zDeM-y|&A#*Tq5R>ar7W!7Nha&xkjRkiz&@5bxw|#EKZpGYYx};Z8}z=@POL6n?Ld2<0a&`BE3~!d8l# zB%yBtXSkBwLW4ax!=0#Ck$jV*n0xk_xv!`?mk3reUM#3IT#bifr_K`y2+4(&G7P7eqYe$ONom1sJLISXvH)&jT#&5-Jxgo{=)HYt2GUlglkJj2ToQyuuwgFGzfnS{hi3n)#XK|HAKb5Vw;N6N6bEX-dupCxj%nxgu>SK$e=T#geDY~ zOb2YF;H~ef9ispag-o;cB-L5&I6Do<(y}cF%Rwor9`i|D%Je}zgqGKG8W;matlmhu z7tsc&OfN}GV4W&8;wKt7XKDuACj+F?_T=@7HyN0SHT6a z*l^eLn8JC6=C!R(&%m+^0wqy?tOBFvlCXM+nLG9VRS_N<{fJBA8H2!uNR*U{?rNJV zIkGLgPuSxuxl79kN`2uvLJSm8hvG5~!{?SupoFD(ODc}L56dC8V{R?|#l2CIbC&EP zT2diR%S!%cYj`V`K}cANs)~fx$(X7E{ZqP$0N|P&9XT^n#r>3LV?A})ks;JnWOobahQ~9iqnv%>nKgBQ~&UY8u5D@_3QW6ud=!2>7GK8Gzu?~gbmP|;X zo#nMgC%O!@AUd95HDBq|6UOR5WMg@4F@lYpAxR>ZptoI5*zXPt4?AfaHR`8g?PCe~ zeAuq*JYA5)GsN`X&*kW_#cg1w$)=dON6i&n*#BjAF^6p-Fd*&Bs-In?@fR|4e7_aD zMSEmm^+gX#WW!)#kvphhPUE3NgHkG6JR^2OLZ^nTFpAOB-E-Xn(vsos=`F4rnn}8S zsRDQUn!B@J#b1;tKn)UX54EF3p}_k{qH7ms5SgNdR3oNk%QJ*qd-}2aPF6+ zJ2fZCwwV_uBEP6aX`&W|%gtS78ki9)%tHAP_B2%dlFHDXwOpxH!9|3B6TFj^po9;d zvJ>&h+0!bK6gjfsd!8?0{h{EComM9CmAOMhFC>T>2vx&OGtp&QQ?ET@^N!(~9%6ZY zS0{_5oix1?u~J`4&?AhaaTFhjV~h0&E$~8^Onv{%$o5U$vx_rL#k;TBp9+g4hQ*Xn z7Oq~qQ}?eBb=CUW+F#CL1CFThBSqi-Qt3gjl+iIa7W(-I|Fe5a=Iirm*8$L5lbt zEQ#rfcqSmAvt1z3i~fA1N=+~`mxk(03EX)+yOq!XOEt~~8GOqx_a1j}QxCse4oT+P zVibs@5)^8~fMxpu(Ugi|=SZ{GKQb$d^|4w8Gu1wU;veRe0c>{{)xxqhZ-CE-D)ZNj z#^kvXGG6;=3~(bDX{Hq^DQA-wc-g9MoFAEnw)>lg%cw$D7L8XUu3^~9ly$D&offl| zaZNinD=A@v&>}&3-r95D<3_MH3-lz8t%K8>)*r$Oo6ng@mY=DzdB^c*&FMn2ct|u9 z-|Mr(>o`Zuc94ueUVKU|>mhmxTj)-;)n@Uh!sDss<>j#OaCK-VjQscKD0!AQrrae3a|5rI{rrhdnPb`@l*QA}=~%n+ zGv@P4)gXAv4X&gk_9%z|T{@Hag(Y&Ph^nOiSJ?P(l2N#ac~`~-$i2p5 zwP2TD4&6SwV6z!m;Ko>z*g@Ref|>1-jY5QYCk(wOba{l;uo{Dq>}+->7MFAWAAy~5 zms=+}D?~R*v}@4Y<2QE8Ke_ldr{b3G_EWzf{1&sT&LeGC>3%21?5(yUXFbKxbFn3A zKF|4S)7T>CRnv82fA#jX5ZqGCQHU@a9j)}EVhwy(D;7M#AAD3#s(yLTIi9${?F{vZ z?9XskvtySJ{6w~aGGlpe*d4PHqiSW<6l{I7#HfOVQuz2=?uP@?WL02;2> zH1B$U9K@k^b)ByTWg4>qW_R7TCxTS)9nl`kTrSoiG9)lC?)$;Vg<2>>cQ_nQw8;XY z!-3ruuEbv)Xa6n*$?E=Lm%m2m%BXE0oY)Hzzkez(*;8X#Gc!wk)b68)Znvd>bguB9 zo|<|KPIcQzhRs^rdNVw{qVtaR+s7}-!f1C zB_sTQ?I2dvxN`BQcU?Asf9e)ksRT~6Q)@T{-(;iHtv z`Vf4+>l@6gXB+!3EJ6+CB}>bpbn%K&P=7^^m)Ux8GVtp`#&TAZQp(Wj(N3fs_#jz{ zSG{3A`HWt3$=or!+ngbm=DCV9n}HC&nrLF0X~=2E7vGb!Kj8&b(zMOiInDqhXxr+2HL_B z%W6c^q5MR>7FERBpNz6cs{;x6WzZ%cDVRh}Sc)IYGz{y}hgi%9t@UX&rK^nrXj>48 zKiQ%|MgjRf2i#bKnR@kjf*D|2A;-V|)*^z_frd1A5w6}dJ6uJ|NVt$=zHYeh+6t<) zmF(v>GZmGXkG8}eF?VvA*P+UWLv}1ezpxE25+-vJZOBntcVVd0 z<%QUm1Y~9Hlz0Yec#dhEPS;=1&%`>2{OJ09S3aTx_AD)o)u*_l`GADgD6MutH)L}! zt4)Vq)4WGK`6ftvN4@3_Zq(oBFp`v=iqQfLm4+;A^ikn2uilalo@tkDVKAK6ysd~d zi2ht=VW)Q1$}p9W;Cwx}d0*^n2atB)Ua$2TSrtPRK&RI8 zs*$|;*hl zS{3f z%fdro6?p{4cgVSw0R8ae*~;K7d(Ef1M{xG0u6BB(`sYUIh_=)6E~v#kjp4CR#cvny ztQHyjEPuC>=ndW9Rq7i#KZO}S&$o+*FR$p4 z88Vl5w8Ke`z?SO{t}vh(s6St|i9bVBS$FX(3GvPAhW0D2ZYn*uDNrux zlh9U7>{+;9CqS(4f8` zQsh#y2A0He3mxx!vA2|lD4BQEs*spY4LFl7M49v3!zZu=&Jn8N+EgQb%__zmB*2w* ztiINu|Lv~=EB6*Bi#R>0WAf zdTG9+59q^~ok}6QTfh;*z_1vB^|>B%N;w}FHJ%xhxHcxyt$7*~2SHuUle4-YSsaHB z2Vc#r&+JUIozH9SQz!MK6_qiQ&L%9~IG!-@(P`tY0}bZF_a6f3T^x)g0#Rbs4&`wK z7|Rh769yM?P-;9}e%=r-3_TeI#YTziHbJXa<|s@aS}fngLXx8D(zl3b@u;A-*n1YM zw;ELk2|8iEpj*b5S~OB*zZO?dDqJ{r8f`5n=8xL1^3%IO2WS?)ts zj%v{qMwji~20UP&V1y(QvFRnAF;BfHaZ15cKVErH0&oE(JN<{JaXO&qRj#5H)v0GrQ6MMqSynrAAZZ za9OpCSMw({dZi0;Hz{^^$|8W_AxmG6aF;%(tw{!n^nUFb{Wu1BbGr8LGPM?4db zetmE)33A3>ykmU!q%m>F4XbUl;&*FH6xYGTCgg${T9=-w0goF2i?^fDs~N-m`EEUM zcjF17V6JcpNit!RJAHg&k`tMhaOBlfPr2(8F!%kRp0>QMt!*i%X5)Jp-4Bjv$-_7K z6PMp4X}QL$dcm|^%J*$J-Y}S-Uw%D!LJi?>2zBDO+PH>iiwQ0%u6fM*Mw9y=+E zjAIlYiHvt^>i9U6|6p^-wkHb5t~PJn^iI1(R&y&dJv}F~-QU(Kzu?_Yjh`egW7(OD zSuW{saBp6_yTM1F{;7^Ph&3xC<&cEaZ%_ZK-j+cU4TU&d>J4Q*AJ%xhdWVfJKl~u0 zu17b+5nDH*VD@o%K32SnSUZ^inzUUa(Q zSbyzg@qv3SNHgi7U=jCkxVw6_f3JJcuzhKvY!Ld6gKznnycF+ojY_!E@NdD>PDPSu z(9__jx#IojnU`{q0lK4$9?cG){yng;WpcpYHgGJsT(~VOq@{xM3h^fn z#t?C5F%}-6=Lw~)h#kPz1iHO{V5Z?7454=REm{(=RBPMo1=`GQ_PHG>p3pI~+S&C@ z`turht-3iwJ_A`?CST*g z|3STZ1X^=J<Bb4R};g5!?v`mrQMF-?YzJb-X7 zv$b#ohaNjdz%K)uPoby0SUpAx6*cSgXtTwkh7#oL0guKXF2W=g1nHPPBg*wO8`jVwV8-KIisa?d{i_r=R!iJJhwd9YLkNfu8e`!Qi)I7aJou?v=`zzvV zE8F|D>S9Y*9t9!JwGvI?wzeXuTB~0_GeNpo(L`$(p zrMTf{Qn0~o1D8btl^tZA>IOYw-v`s3`DA_k`Sa(il90A*4m1yq-P^T(uetjB;oR`F z94FpEwftc^{8Th-i=|wVpCRYNKyNsTp>(%MO)65weQW1{ATRbUe|6++9qlYfAoFG> z@@k3|=|;reVZ-E$H})IcbXcpp?eyjr@+)ye$8q_bV$4`)9QOg8_GeUeu|@5Kz>OH{ z8ISnkV?;(S_C7cU$vWm#$pwt=lg+w%mM0o-{Yg8DFPeRT`0IXVS;`Iicai*tx7UfvQ-tC=r?Y7Q84a|_lMF1n4uI6-0|#8nZDu9^I=i_AkzZ?f z!tXPX$HvIsEjitF9;~@tQsf-6yKbM&Yu8C>X^zD+W4GTOv^P)&b+FZ5`GtN^3zJJ= z3rN-jiINn<(=7@5dC0W=4o`b?q1wMcPDOuSwdEO2#HLTFc~g=Tyvkv7EJ_*Z=METzU3L6OB_$V zY0H0n;|q0S`Mt<0EOd#_BxlF+Z$ zc`zZ7t+O|W7$9jw(hd743eFquTVuH2lnWAdwhTyJ^46%=9PVi-XS)|WyqzT!&q{c( zyB1xEuLvYnKfqZ2_@e4LK!&|{1TP0)hO4im#jcjDL&Sdaybd=00uXV#rNQ@qcwjcC u{yRVT=lK7}5iZ8AliL3C6dT?NWxekIuAQNN`ySY^;o23;KZ-BkeE1(|Y^7WP literal 0 HcmV?d00001 diff --git a/figures/canary_deployment.svg b/figures/canary_deployment.svg new file mode 100644 index 000000000..a6e8bbee9 --- /dev/null +++ b/figures/canary_deployment.svg @@ -0,0 +1,2092 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.1 + 2020-01-16 10:05:46 +0000 + + + Diagram 4 + + Layer 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/figures/shadow_testing.svg b/figures/shadow_testing.svg new file mode 100644 index 000000000..565c64d1f --- /dev/null +++ b/figures/shadow_testing.svg @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.1 + 2020-01-16 10:05:46 +0000 + + + Diagram 5 + + Layer 1 + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + zone external - grey + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + users + + + + + + Shape_1_ + + + + + + + + + + + + + + + + + + + network load balancer + + + + + + + + + + + + + + + + + + + + + + + + + + From 1fdfa2d2147d4170b5944fddab0d18f695a05239 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Thu, 27 Jun 2024 12:32:36 +0200 Subject: [PATCH 3/4] more text --- mkdocs.yml | 24 +++++----- requirements.txt | 1 + s7_deployment/deployment_testing.md | 71 +++++++++++++++++++++++++---- 3 files changed, 77 insertions(+), 19 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 1dfbb546a..64f3bec10 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,6 +64,7 @@ plugins: - search - glightbox - same-dir + - inline-svg - git-revision-date-localized: enable_creation_date: true - exclude: @@ -128,23 +129,24 @@ nav: - M22 - Local Deployment: s7_deployment/local_deployment.md - M23 - Cloud Deployment: s7_deployment/cloud_deployment.md - M24 - API Testing: s7_deployment/testing_apis.md + - M25 - Deployment Testing: s7_deployment/deployment_testing.md - S8 - Monitoring 📊: - s8_monitoring/README.md - - M25 - Data Drifting: s8_monitoring/data_drifting.md - - M26 - System Monitoring: s8_monitoring/monitoring.md + - M26 - Data Drifting: s8_monitoring/data_drifting.md + - M27 - System Monitoring: s8_monitoring/monitoring.md - S9 - Scalable applications ⚖️: - s9_scalable_applications/README.md - - M27 - Distributed Data Loading: s9_scalable_applications/data_loading.md - - M28 - Distributed Training: s9_scalable_applications/distributed_training.md - - M29 - Scalable Inference: s9_scalable_applications/inference.md + - M28 - Distributed Data Loading: s9_scalable_applications/data_loading.md + - M29 - Distributed Training: s9_scalable_applications/distributed_training.md + - M30 - Scalable Inference: s9_scalable_applications/inference.md - S10 - Extra 🔥: - s10_extra/README.md - - M30 - Command Line Interfaces: s10_extra/cli.md - - M31 - Documentation: s10_extra/documentation.md - - M32 - Hyperparameter optimization: s10_extra/hyperparameters.md - - M33 - High Performance Clusters: s10_extra/high_performance_clusters.md - - M34 - Frontend: s10_extra/frontend.md - - M35 - ML deployment: s10_extra/ml_deployment.md + - M31 - Command Line Interfaces: s10_extra/cli.md + - M32 - Documentation: s10_extra/documentation.md + - M33 - Hyperparameter optimization: s10_extra/hyperparameters.md + - M34 - High Performance Clusters: s10_extra/high_performance_clusters.md + - M35 - Frontend: s10_extra/frontend.md + - M36 - ML deployment: s10_extra/ml_deployment.md # - M35 - Designing Pipelines: s10_extra/design.md # - M37 - Workflow orchestration: s10_extra/orchestration.md # - M38 - Kubernetes: s10_extra/kubernetes.md diff --git a/requirements.txt b/requirements.txt index 0d4ad3613..4040a3b5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ pymdown-extensions>=1.1.1 mkdocs-same-dir>=0.1.2 mkdocs-git-revision-date-localized-plugin>=1.2.0 mkdocs-exclude>=1.0.2 +mkdocs-plugin-inline-svg>=0.1.0 # Developer stuff ruff>=0.4.7 diff --git a/s7_deployment/deployment_testing.md b/s7_deployment/deployment_testing.md index e9bacc65f..6b52560b1 100644 --- a/s7_deployment/deployment_testing.md +++ b/s7_deployment/deployment_testing.md @@ -1,20 +1,57 @@ -![Logo](../figures/icons/cloudrun.png){ align=right width="130"} +![Logo](../figures/icons/gcp.png){ align=right width="130"} # Deployment Testing -https://cloud.google.com/architecture/application-deployment-and-testing-strategies +In the module on [testing of APIs](testing_apis.md) we learned how to write integration tests for our APIs and how to +run loadtests to see how our API behaves under different loads. In this module we are going to learn about a few +different deployment testing strategies that can be used to ensure that our application is working as expected *before* +we deploy it to production. Deployment testing is designed to minimize risk, enhance performance, and ensure a smooth +transition from development to production. Being able to choose and execute the correct deployment testing strategy is +even more important within machine learning projects as we tend to update our models more frequently than traditional +software projects, thus requiring more frequent deployments. -The testing patterns discussed in this section are typically used to validate a service's reliability and stability over -a reasonable period under a realistic level of concurrency and load. - -In today's dynamic software development landscape, ensuring seamless and reliable application deployment is crucial for maintaining user satisfaction and operational efficiency. This module will introduce you to various deployment testing patterns, each designed to minimize risk, enhance performance, and ensure a smooth transition from development to production. Whether you're aiming for zero downtime, incremental feature releases, or robust rollback capabilities, understanding these strategies will empower you to implement effective deployment workflows. Join us as we explore canary deployments, blue-green deployments, feature toggles, and more, equipping you with the knowledge to optimize your deployment processes and deliver high-quality software with confidence. +In general we recommend you start out with reading this +[page](https://cloud.google.com/architecture/application-deployment-and-testing-strategies) from GCP on both application +deployment and testing strategies. It is a good read on the different metrics we can use to evaluate our deployment +(down time, rollback duration, etc.) and the different deployment strategies we can use. In the following we are going +to be looking at the three testing methods +* A/B testing +* Canary deployment +* Shadow deployment ## A/B testing +In software development, [A/B testing](https://en.wikipedia.org/wiki/A/B_testing) is a method of comparing two versions +of a web page or application against each other to determine which one performs better. A/B testing is a form of +statistical hypothesis testing with two variants + + + + +
+![Image](../figures/a_b_testing.svg) +
+ Image credit +
+
+ +[text](https://www.surveymonkey.com/mp/ab-testing-significance-calculator/) + ### ❔ Exercises -1. Geolocation +In the exercises we are going to perform two different kinds of A/B testing. The first one is going to be a simple +A/B test where we are going to test two different versions of the same service. The second + +1. There are a multiple ways to implement A/B testing. ```python from fastapi import FastAPI, Request, HTTPException @@ -59,6 +96,14 @@ In today's dynamic software development landscape, ensuring seamless and reliabl ## Canary deployment +
+![Image](../figures/canary_deployment.svg) +
+ Image credit +
+
+ + ### ❔ Exercises Follow [these](https://cloud.google.com/architecture/implementing-cloud-run-canary-deployments-git-branches-cloud-build) @@ -72,6 +117,14 @@ instructions to implement a canary deployment using Git branches and Cloud Build ## Shadow deployment +
+![Image](../figures/shadow_testing.svg) +
+ Image credit +
+
+ + ### ❔ Exercises 1. Google Run does not naturally support shadow deployments, because its loadbalancer requires that the traffic adds up @@ -121,4 +174,6 @@ instructions to implement a canary deployment using Git branches and Cloud Build --------------- | -------------- | -------------------------------- | -------------------------------------- | ----------------- | -------------------------------------- | A/B testing | No | No | Yes | Short | No | Canary deployment | Yes | Yes | Yes | Short | Yes | - Shadow deployment | Yes | No | Yes | Short | Yes | \ No newline at end of file + Shadow deployment | Yes | No | Yes | Short | Yes | + +This ends the deployment testing module. From cd765bf6d569c56d1ad120f5e78c874e039829ab Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Thu, 27 Jun 2024 12:36:36 +0200 Subject: [PATCH 4/4] add table --- s7_deployment/deployment_testing.md | 42 ++++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/s7_deployment/deployment_testing.md b/s7_deployment/deployment_testing.md index 6b52560b1..99dd6ebe5 100644 --- a/s7_deployment/deployment_testing.md +++ b/s7_deployment/deployment_testing.md @@ -3,15 +3,15 @@ # Deployment Testing In the module on [testing of APIs](testing_apis.md) we learned how to write integration tests for our APIs and how to -run loadtests to see how our API behaves under different loads. In this module we are going to learn about a few +run loadtests to see how our API behaves under different loads. In this module we are going to learn about a few different deployment testing strategies that can be used to ensure that our application is working as expected *before* we deploy it to production. Deployment testing is designed to minimize risk, enhance performance, and ensure a smooth transition from development to production. Being able to choose and execute the correct deployment testing strategy is even more important within machine learning projects as we tend to update our models more frequently than traditional software projects, thus requiring more frequent deployments. -In general we recommend you start out with reading this -[page](https://cloud.google.com/architecture/application-deployment-and-testing-strategies) from GCP on both application +In general we recommend you start out with reading this +[page](https://cloud.google.com/architecture/application-deployment-and-testing-strategies) from GCP on both application deployment and testing strategies. It is a good read on the different metrics we can use to evaluate our deployment (down time, rollback duration, etc.) and the different deployment strategies we can use. In the following we are going to be looking at the three testing methods @@ -22,34 +22,44 @@ to be looking at the three testing methods ## A/B testing -In software development, [A/B testing](https://en.wikipedia.org/wiki/A/B_testing) is a method of comparing two versions -of a web page or application against each other to determine which one performs better. A/B testing is a form of +In software development, [A/B testing](https://en.wikipedia.org/wiki/A/B_testing) is a method of comparing two versions +of a web page or application against each other to determine which one performs better. A/B testing is a form of statistical hypothesis testing with two variants
![Image](../figures/a_b_testing_example.png) -
-In this case we are randomly A/B testing if the color and style of the `Learn more` button affects the click rate. In +
+In this case we are randomly A/B testing if the color and style of the `Learn more` button affects the click rate. In this hypothetical example, the green button has a 20% higher click rate than the blue button and is therefore the preferred choice for the final design. - Image credit + Image credit
![Image](../figures/a_b_testing.svg) -
- Image credit +
+ Image credit
[text](https://www.surveymonkey.com/mp/ab-testing-significance-calculator/) + +Assumed Distribution | Example case | Standard test +-------------------- | ------------ | ------------- +Gaussian | Average revenue per user | t-test +Binomial | Click-through rate | Fisher's exact test +Poisson | Number of purchases | Chi-squared test +Multinomial | User preferences | Chi-squared test +Unknown | Time to purchase | Mann-Whitney U test + + ### ❔ Exercises In the exercises we are going to perform two different kinds of A/B testing. The first one is going to be a simple -A/B test where we are going to test two different versions of the same service. The second +A/B test where we are going to test two different versions of the same service. The second 1. There are a multiple ways to implement A/B testing. @@ -98,8 +108,8 @@ A/B test where we are going to test two different versions of the same service.
![Image](../figures/canary_deployment.svg) -
- Image credit +
+ Image credit
@@ -119,8 +129,8 @@ instructions to implement a canary deployment using Git branches and Cloud Build
![Image](../figures/shadow_testing.svg) -
- Image credit +
+ Image credit
@@ -154,7 +164,7 @@ instructions to implement a canary deployment using Git branches and Cloud Build "response": response.json() } ``` - + 2. Because the loadbalancer is just a simple Python script lets just deploy it to Cloud Functions instead of Cloud Run. Create a new Cloud Function and deploy the script.
+![Image](../figures/a_b_testing_example.png) +
+In this case we are randomly A/B testing if the color and style of the `Learn more` button affects the click rate. In +this hypothetical example, the green button has a 20% higher click rate than the blue button and is therefore the +preferred choice for the final design. + Image credit +
+