<!DOCTYPE html> <html> <head> <meta name="databricks-html-version" content="1"> <title>050_CaseStudy_HighOrderSpectralClustering - Databricks</title> <meta charset="utf-8"> <meta name="google" content="notranslate"> <meta http-equiv="Content-Language" content="en"> <meta http-equiv="Content-Type" content="text/html; charset=UTF8"> <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Source+Code+Pro:400,700"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/css/bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/lib/jquery-ui-bundle/jquery-ui.min.css"> <link rel="stylesheet" type="text/css" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/main.css"> <link rel="stylesheet" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/css/print.css" media="print"> <link rel="icon" type="image/png" href="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/img/favicon.ico"/> <script>window.settings = {"sparkDocsSearchGoogleCx":"004588677886978090460:_rj0wilqwdm","dbcForumURL":"http://forums.databricks.com/","dbfsS3Host":"https://databricks-prod-storage-sydney.s3.amazonaws.com","enableThirdPartyApplicationsUI":false,"enableClusterAcls":false,"notebookRevisionVisibilityHorizon":0,"enableTableHandler":true,"isAdmin":true,"enableLargeResultDownload":false,"nameAndEmail":"Raazesh Sainudiin (r.sainudiin@math.canterbury.ac.nz)","enablePresentationTimerConfig":true,"enableFullTextSearch":true,"enableElasticSparkUI":true,"clusters":true,"hideOffHeapCache":false,"applications":false,"useStaticGuide":false,"fileStoreBase":"FileStore","configurableSparkOptionsSpec":[{"keyPattern":"spark\\.kryo(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.kryo.*","valuePatternDisplay":"*","description":"Configuration options for Kryo serialization"},{"keyPattern":"spark\\.io\\.compression\\.codec","valuePattern":"(lzf|snappy|org\\.apache\\.spark\\.io\\.LZFCompressionCodec|org\\.apache\\.spark\\.io\\.SnappyCompressionCodec)","keyPatternDisplay":"spark.io.compression.codec","valuePatternDisplay":"snappy|lzf","description":"The codec used to compress internal data such as RDD partitions, broadcast variables and shuffle outputs."},{"keyPattern":"spark\\.serializer","valuePattern":"(org\\.apache\\.spark\\.serializer\\.JavaSerializer|org\\.apache\\.spark\\.serializer\\.KryoSerializer)","keyPatternDisplay":"spark.serializer","valuePatternDisplay":"org.apache.spark.serializer.JavaSerializer|org.apache.spark.serializer.KryoSerializer","description":"Class to use for serializing objects that will be sent over the network or need to be cached in serialized form."},{"keyPattern":"spark\\.rdd\\.compress","valuePattern":"(true|false)","keyPatternDisplay":"spark.rdd.compress","valuePatternDisplay":"true|false","description":"Whether to compress serialized RDD partitions (e.g. for StorageLevel.MEMORY_ONLY_SER). Can save substantial space at the cost of some extra CPU time."},{"keyPattern":"spark\\.speculation","valuePattern":"(true|false)","keyPatternDisplay":"spark.speculation","valuePatternDisplay":"true|false","description":"Whether to use speculation (recommended off for streaming)"},{"keyPattern":"spark\\.es(\\.[^\\.]+)+","valuePattern":".*","keyPatternDisplay":"spark.es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"es(\\.([^\\.]+))+","valuePattern":".*","keyPatternDisplay":"es.*","valuePatternDisplay":"*","description":"Configuration options for ElasticSearch"},{"keyPattern":"spark\\.(storage|shuffle)\\.memoryFraction","valuePattern":"0?\\.0*([1-9])([0-9])*","keyPatternDisplay":"spark.(storage|shuffle).memoryFraction","valuePatternDisplay":"(0.0,1.0)","description":"Fraction of Java heap to use for Spark's shuffle or storage"},{"keyPattern":"spark\\.streaming\\.backpressure\\.enabled","valuePattern":"(true|false)","keyPatternDisplay":"spark.streaming.backpressure.enabled","valuePatternDisplay":"true|false","description":"Enables or disables Spark Streaming's internal backpressure mechanism (since 1.5). This enables the Spark Streaming to control the receiving rate based on the current batch scheduling delays and processing times so that the system receives only as fast as the system can process. Internally, this dynamically sets the maximum receiving rate of receivers. This rate is upper bounded by the values `spark.streaming.receiver.maxRate` and `spark.streaming.kafka.maxRatePerPartition` if they are set."},{"keyPattern":"spark\\.streaming\\.receiver\\.maxRate","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.receiver.maxRate","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which each receiver will receive data. Effectively, each stream will consume at most this number of records per second. Setting this configuration to 0 or a negative number will put no limit on the rate. See the deployment guide in the Spark Streaming programing guide for mode details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRatePerPartition","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRatePerPartition","valuePatternDisplay":"numeric","description":"Maximum rate (number of records per second) at which data will be read from each Kafka partition when using the Kafka direct stream API introduced in Spark 1.3. See the Kafka Integration guide for more details."},{"keyPattern":"spark\\.streaming\\.kafka\\.maxRetries","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.kafka.maxRetries","valuePatternDisplay":"numeric","description":"Maximum number of consecutive retries the driver will make in order to find the latest offsets on the leader of each partition (a default value of 1 means that the driver will make a maximum of 2 attempts). Only applies to the Kafka direct stream API introduced in Spark 1.3."},{"keyPattern":"spark\\.streaming\\.ui\\.retainedBatches","valuePattern":"^([0-9]{1,})$","keyPatternDisplay":"spark.streaming.ui.retainedBatches","valuePatternDisplay":"numeric","description":"How many batches the Spark Streaming UI and status APIs remember before garbage collecting."}],"enableReactNotebookComments":true,"enableResetPassword":true,"enableJobsSparkUpgrade":true,"sparkVersions":[{"key":"1.3.x-ubuntu15.10","displayName":"Spark 1.3.0","packageLabel":"spark-1.3-jenkins-ip-10-30-9-162-U0c2673ac85-Sa2ee4664b2-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.4.x-ubuntu15.10","displayName":"Spark 1.4.1","packageLabel":"spark-1.4-jenkins-ip-10-30-9-162-U0c2673ac85-S33a1e4b9c6-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"1.6.x-ubuntu15.10","displayName":"Spark 1.6.0","packageLabel":"spark-1.6-jenkins-ip-10-30-9-162-U0c2673ac85-Scabba801f3-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},{"key":"master","displayName":"Spark master (dev)","packageLabel":"","upgradable":true,"deprecated":false,"customerVisible":false}],"enableRestrictedClusterCreation":false,"enableFeedback":false,"defaultNumWorkers":8,"serverContinuationTimeoutMillis":10000,"driverStderrFilePrefix":"stderr","driverStdoutFilePrefix":"stdout","enableSparkDocsSearch":true,"prefetchSidebarNodes":true,"sparkHistoryServerEnabled":true,"sanitizeMarkdownHtml":true,"enableIPythonImportExport":true,"enableNotebookHistoryDiffing":true,"branch":"2.12.3","accountsLimit":-1,"enableNotebookGitBranching":true,"local":false,"displayDefaultContainerMemoryGB":6,"deploymentMode":"production","useSpotForWorkers":false,"enableUserInviteWorkflow":false,"enableStaticNotebooks":true,"dbcGuideURL":"#workspace/databricks_guide/00 Welcome to Databricks","enableCssTransitions":true,"pricingURL":"https://databricks.com/product/pricing","enableClusterAclsConfig":false,"orgId":0,"enableNotebookGitVersioning":true,"files":"files/","enableDriverLogsUI":true,"disableLegacyDashboards":false,"enableWorkspaceAclsConfig":true,"dropzoneMaxFileSize":4096,"enableNewDashboardViews":false,"driverLog4jFilePrefix":"log4j","enableMavenLibraries":true,"displayRowLimit":1000,"defaultSparkVersion":{"key":"1.5.x-ubuntu15.10","displayName":"Spark 1.5.2","packageLabel":"spark-1.5-jenkins-ip-10-30-9-162-U0c2673ac85-S5917a1044d-2016-02-09-02:05:59.455061","upgradable":true,"deprecated":false,"customerVisible":true},"clusterPublisherRootId":5,"enableLatestJobRunResultPermalink":true,"disallowAddingAdmins":false,"enableSparkConfUI":true,"enableOrgSwitcherUI":false,"clustersLimit":-1,"enableJdbcImport":true,"logfiles":"logfiles/","enableWebappSharding":false,"enableClusterDeltaUpdates":true,"csrfToken":"4c37e4ff-a908-4b05-8c61-2b76819fa34c","useFixedStaticNotebookVersionForDevelopment":false,"enableBasicReactDialogBoxes":true,"requireEmailUserName":true,"enableDashboardViews":false,"dbcFeedbackURL":"http://feedback.databricks.com/forums/263785-product-feedback","enableWorkspaceAclService":true,"someName":"Raazesh Sainudiin","enableWorkspaceAcls":true,"gitHash":"0c2673ac858e227cad536fdb45d140aeded238db","userFullname":"Raazesh Sainudiin","enableClusterCreatePage":false,"enableImportFromUrl":true,"enableMiniClusters":false,"enableWebSocketDeltaUpdates":true,"enableDebugUI":false,"showHiddenSparkVersions":false,"allowNonAdminUsers":true,"userId":100005,"dbcSupportURL":"","staticNotebookResourceUrl":"https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/","enableSparkPackages":true,"enableHybridClusterType":false,"enableNotebookHistoryUI":true,"availableWorkspaces":[{"name":"Workspace 0","orgId":0}],"enableFolderHtmlExport":true,"enableSparkVersionsUI":true,"databricksGuideStaticUrl":"","enableHybridClusters":true,"notebookLoadingBackground":"#fff","enableNewJobRunDetailsPage":true,"enableDashboardExport":true,"user":"r.sainudiin@math.canterbury.ac.nz","enableServerAutoComplete":true,"enableStaticHtmlImport":true,"defaultMemoryPerContainerMB":6000,"enablePresenceUI":true,"tablesPublisherRootId":7,"enableNewInputWidgetUI":false,"accounts":true,"enableNewProgressReportUI":true,"defaultCoresPerContainer":4};</script> <script>var __DATABRICKS_NOTEBOOK_MODEL = {"version":"NotebookV1","origId":124218,"name":"050_CaseStudy_HighOrderSpectralClustering","language":"scala","commands":[{"version":"CommandV1","origId":124220,"guid":"29b3f866-e961-4143-96e7-38337bb5c9c8","subtype":"command","commandType":"auto","position":0.125,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n## Course Project - High Order Spectral Clustering\n### by Xin Zhao\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536403901E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2084c54b-d8c0-4744-9b86-74e7d410187f"},{"version":"CommandV1","origId":140757,"guid":"2087d2ba-4342-4f52-9031-869ff0d47f07","subtype":"command","commandType":"auto","position":0.1875,"command":"%md\nThe [html source url](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/db/studentProjects/04_XinZhao/050_CaseStudy_HighOrderSpectralClustering.html) of this databricks notebook and its recorded Uji :\n\n[](https://www.youtube.com/v/zJirlHAV6YU?rel=0&autoplay=1&modestbranding=1&start=4019&end=4677)\n","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"7dca0670-4983-4568-a102-b9b6554b9662"},{"version":"CommandV1","origId":124221,"guid":"894cd0ba-b46b-428f-8a55-9a25a07b8b57","subtype":"command","commandType":"auto","position":0.25,"command":"%md\n#Example of High Order Spectral Clustering\n[Iris flower data set](https://en.wikipedia.org/wiki/Iris_flower_data_set):\nThe data set consists of 50 samples from each of three species of Iris (Iris setosa, Iris versicolor and Iris virginica). \n\nsetosa: <img src=\"https://upload.wikimedia.org/wikipedia/commons/5/56/Kosaciec_szczecinkowaty_Iris_setosa.jpg\" width=\"300\">,\nversicolor: <img src=\"https://upload.wikimedia.org/wikipedia/commons/4/41/Iris_versicolor_3.jpg\" width=\"400\">, and\nvirginica: <img src=\"https://upload.wikimedia.org/wikipedia/commons/9/9f/Iris_virginica.jpg\" width=\"400\">\n\nBased on Fisher's linear discriminant model, this data set became a typical test case for many statistical classification techniques in machine learning such as support vector machines.\nThe use of this data set in cluster analysis however is uncommon, since the data set only contains two clusters with rather obvious separation. One of the clusters contains Iris setosa, while the other cluster contains both Iris virginica and Iris versicolor and is not separable without the species information Fisher used. ","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536403902E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5341be8b-3f7e-48db-a180-574a7a2f6e83"},{"version":"CommandV1","origId":124222,"guid":"4f6790a5-0ed8-4c7a-aae8-638352b9e9c2","subtype":"command","commandType":"auto","position":0.3125,"command":"%md\nThe table below shows the iris data set.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.46553640391E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e3a61d3e-9cd8-414c-a3df-7e35bb9485d5"},{"version":"CommandV1","origId":124223,"guid":"a8909a59-5e5f-4262-a504-1c548f255eae","subtype":"command","commandType":"auto","position":0.375,"command":"val fisheririsDF = sqlContext.table(\"fisheriris\")\ndisplay(fisheririsDF)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[5.1,3.5,1.4,0.2,"'setosa'",0.0],[4.9,3.0,1.4,0.2,"'setosa'",1.0],[4.7,3.2,1.3,0.2,"'setosa'",2.0],[4.6,3.1,1.5,0.2,"'setosa'",3.0],[5.0,3.6,1.4,0.2,"'setosa'",4.0],[5.4,3.9,1.7,0.4,"'setosa'",5.0],[4.6,3.4,1.4,0.3,"'setosa'",6.0],[5.0,3.4,1.5,0.2,"'setosa'",7.0],[4.4,2.9,1.4,0.2,"'setosa'",8.0],[4.9,3.1,1.5,0.1,"'setosa'",9.0],[5.4,3.7,1.5,0.2,"'setosa'",10.0],[4.8,3.4,1.6,0.2,"'setosa'",11.0],[4.8,3.0,1.4,0.1,"'setosa'",12.0],[4.3,3.0,1.1,0.1,"'setosa'",13.0],[5.8,4.0,1.2,0.2,"'setosa'",14.0],[5.7,4.4,1.5,0.4,"'setosa'",15.0],[5.4,3.9,1.3,0.4,"'setosa'",16.0],[5.1,3.5,1.4,0.3,"'setosa'",17.0],[5.7,3.8,1.7,0.3,"'setosa'",18.0],[5.1,3.8,1.5,0.3,"'setosa'",19.0],[5.4,3.4,1.7,0.2,"'setosa'",20.0],[5.1,3.7,1.5,0.4,"'setosa'",21.0],[4.6,3.6,1.0,0.2,"'setosa'",22.0],[5.1,3.3,1.7,0.5,"'setosa'",23.0],[4.8,3.4,1.9,0.2,"'setosa'",24.0],[5.0,3.0,1.6,0.2,"'setosa'",25.0],[5.0,3.4,1.6,0.4,"'setosa'",26.0],[5.2,3.5,1.5,0.2,"'setosa'",27.0],[5.2,3.4,1.4,0.2,"'setosa'",28.0],[4.7,3.2,1.6,0.2,"'setosa'",29.0],[4.8,3.1,1.6,0.2,"'setosa'",30.0],[5.4,3.4,1.5,0.4,"'setosa'",31.0],[5.2,4.1,1.5,0.1,"'setosa'",32.0],[5.5,4.2,1.4,0.2,"'setosa'",33.0],[4.9,3.1,1.5,0.2,"'setosa'",34.0],[5.0,3.2,1.2,0.2,"'setosa'",35.0],[5.5,3.5,1.3,0.2,"'setosa'",36.0],[4.9,3.6,1.4,0.1,"'setosa'",37.0],[4.4,3.0,1.3,0.2,"'setosa'",38.0],[5.1,3.4,1.5,0.2,"'setosa'",39.0],[5.0,3.5,1.3,0.3,"'setosa'",40.0],[4.5,2.3,1.3,0.3,"'setosa'",41.0],[4.4,3.2,1.3,0.2,"'setosa'",42.0],[5.0,3.5,1.6,0.6,"'setosa'",43.0],[5.1,3.8,1.9,0.4,"'setosa'",44.0],[4.8,3.0,1.4,0.3,"'setosa'",45.0],[5.1,3.8,1.6,0.2,"'setosa'",46.0],[4.6,3.2,1.4,0.2,"'setosa'",47.0],[5.3,3.7,1.5,0.2,"'setosa'",48.0],[5.0,3.3,1.4,0.2,"'setosa'",49.0],[7.0,3.2,4.7,1.4,"'versicolor'",50.0],[6.4,3.2,4.5,1.5,"'versicolor'",51.0],[6.9,3.1,4.9,1.5,"'versicolor'",52.0],[5.5,2.3,4.0,1.3,"'versicolor'",53.0],[6.5,2.8,4.6,1.5,"'versicolor'",54.0],[5.7,2.8,4.5,1.3,"'versicolor'",55.0],[6.3,3.3,4.7,1.6,"'versicolor'",56.0],[4.9,2.4,3.3,1.0,"'versicolor'",57.0],[6.6,2.9,4.6,1.3,"'versicolor'",58.0],[5.2,2.7,3.9,1.4,"'versicolor'",59.0],[5.0,2.0,3.5,1.0,"'versicolor'",60.0],[5.9,3.0,4.2,1.5,"'versicolor'",61.0],[6.0,2.2,4.0,1.0,"'versicolor'",62.0],[6.1,2.9,4.7,1.4,"'versicolor'",63.0],[5.6,2.9,3.6,1.3,"'versicolor'",64.0],[6.7,3.1,4.4,1.4,"'versicolor'",65.0],[5.6,3.0,4.5,1.5,"'versicolor'",66.0],[5.8,2.7,4.1,1.0,"'versicolor'",67.0],[6.2,2.2,4.5,1.5,"'versicolor'",68.0],[5.6,2.5,3.9,1.1,"'versicolor'",69.0],[5.9,3.2,4.8,1.8,"'versicolor'",70.0],[6.1,2.8,4.0,1.3,"'versicolor'",71.0],[6.3,2.5,4.9,1.5,"'versicolor'",72.0],[6.1,2.8,4.7,1.2,"'versicolor'",73.0],[6.4,2.9,4.3,1.3,"'versicolor'",74.0],[6.6,3.0,4.4,1.4,"'versicolor'",75.0],[6.8,2.8,4.8,1.4,"'versicolor'",76.0],[6.7,3.0,5.0,1.7,"'versicolor'",8.589934592E9],[6.0,2.9,4.5,1.5,"'versicolor'",8.589934593E9],[5.7,2.6,3.5,1.0,"'versicolor'",8.589934594E9],[5.5,2.4,3.8,1.1,"'versicolor'",8.589934595E9],[5.5,2.4,3.7,1.0,"'versicolor'",8.589934596E9],[5.8,2.7,3.9,1.2,"'versicolor'",8.589934597E9],[6.0,2.7,5.1,1.6,"'versicolor'",8.589934598E9],[5.4,3.0,4.5,1.5,"'versicolor'",8.589934599E9],[6.0,3.4,4.5,1.6,"'versicolor'",8.5899346E9],[6.7,3.1,4.7,1.5,"'versicolor'",8.589934601E9],[6.3,2.3,4.4,1.3,"'versicolor'",8.589934602E9],[5.6,3.0,4.1,1.3,"'versicolor'",8.589934603E9],[5.5,2.5,4.0,1.3,"'versicolor'",8.589934604E9],[5.5,2.6,4.4,1.2,"'versicolor'",8.589934605E9],[6.1,3.0,4.6,1.4,"'versicolor'",8.589934606E9],[5.8,2.6,4.0,1.2,"'versicolor'",8.589934607E9],[5.0,2.3,3.3,1.0,"'versicolor'",8.589934608E9],[5.6,2.7,4.2,1.3,"'versicolor'",8.589934609E9],[5.7,3.0,4.2,1.2,"'versicolor'",8.58993461E9],[5.7,2.9,4.2,1.3,"'versicolor'",8.589934611E9],[6.2,2.9,4.3,1.3,"'versicolor'",8.589934612E9],[5.1,2.5,3.0,1.1,"'versicolor'",8.589934613E9],[5.7,2.8,4.1,1.3,"'versicolor'",8.589934614E9],[6.3,3.3,6.0,2.5,"'virginica'",8.589934615E9],[5.8,2.7,5.1,1.9,"'virginica'",8.589934616E9],[7.1,3.0,5.9,2.1,"'virginica'",8.589934617E9],[6.3,2.9,5.6,1.8,"'virginica'",8.589934618E9],[6.5,3.0,5.8,2.2,"'virginica'",8.589934619E9],[7.6,3.0,6.6,2.1,"'virginica'",8.58993462E9],[4.9,2.5,4.5,1.7,"'virginica'",8.589934621E9],[7.3,2.9,6.3,1.8,"'virginica'",8.589934622E9],[6.7,2.5,5.8,1.8,"'virginica'",8.589934623E9],[7.2,3.6,6.1,2.5,"'virginica'",8.589934624E9],[6.5,3.2,5.1,2.0,"'virginica'",8.589934625E9],[6.4,2.7,5.3,1.9,"'virginica'",8.589934626E9],[6.8,3.0,5.5,2.1,"'virginica'",8.589934627E9],[5.7,2.5,5.0,2.0,"'virginica'",8.589934628E9],[5.8,2.8,5.1,2.4,"'virginica'",8.589934629E9],[6.4,3.2,5.3,2.3,"'virginica'",8.58993463E9],[6.5,3.0,5.5,1.8,"'virginica'",8.589934631E9],[7.7,3.8,6.7,2.2,"'virginica'",8.589934632E9],[7.7,2.6,6.9,2.3,"'virginica'",8.589934633E9],[6.0,2.2,5.0,1.5,"'virginica'",8.589934634E9],[6.9,3.2,5.7,2.3,"'virginica'",8.589934635E9],[5.6,2.8,4.9,2.0,"'virginica'",8.589934636E9],[7.7,2.8,6.7,2.0,"'virginica'",8.589934637E9],[6.3,2.7,4.9,1.8,"'virginica'",8.589934638E9],[6.7,3.3,5.7,2.1,"'virginica'",8.589934639E9],[7.2,3.2,6.0,1.8,"'virginica'",8.58993464E9],[6.2,2.8,4.8,1.8,"'virginica'",8.589934641E9],[6.1,3.0,4.9,1.8,"'virginica'",8.589934642E9],[6.4,2.8,5.6,2.1,"'virginica'",8.589934643E9],[7.2,3.0,5.8,1.6,"'virginica'",8.589934644E9],[7.4,2.8,6.1,1.9,"'virginica'",8.589934645E9],[7.9,3.8,6.4,2.0,"'virginica'",8.589934646E9],[6.4,2.8,5.6,2.2,"'virginica'",8.589934647E9],[6.3,2.8,5.1,1.5,"'virginica'",8.589934648E9],[6.1,2.6,5.6,1.4,"'virginica'",8.589934649E9],[7.7,3.0,6.1,2.3,"'virginica'",8.58993465E9],[6.3,3.4,5.6,2.4,"'virginica'",8.589934651E9],[6.4,3.1,5.5,1.8,"'virginica'",8.589934652E9],[6.0,3.0,4.8,1.8,"'virginica'",8.589934653E9],[6.9,3.1,5.4,2.1,"'virginica'",8.589934654E9],[6.7,3.1,5.6,2.4,"'virginica'",8.589934655E9],[6.9,3.1,5.1,2.3,"'virginica'",8.589934656E9],[5.8,2.7,5.1,1.9,"'virginica'",8.589934657E9],[6.8,3.2,5.9,2.3,"'virginica'",8.589934658E9],[6.7,3.3,5.7,2.5,"'virginica'",8.589934659E9],[6.7,3.0,5.2,2.3,"'virginica'",8.58993466E9],[6.3,2.5,5.0,1.9,"'virginica'",8.589934661E9],[6.5,3.0,5.2,2.0,"'virginica'",8.589934662E9],[6.2,3.4,5.4,2.3,"'virginica'",8.589934663E9],[5.9,3.0,5.1,1.8,"'virginica'",8.589934664E9]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"sepalLength","type":"\"double\""},{"name":"sepalWidth","type":"\"double\""},{"name":"petalLength","type":"\"double\""},{"name":"petalWidth","type":"\"double\""},{"name":"specie","type":"\"string\""},{"name":"id","type":"\"long\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"org.apache.spark.sql.catalyst.analysis.NoSuchTableException","error":"<div class=\"ansiout\">\tat org.apache.spark.sql.hive.client.ClientInterface$$anonfun$getTable$1.apply(ClientInterface.scala:122)\n\tat org.apache.spark.sql.hive.client.ClientInterface$$anonfun$getTable$1.apply(ClientInterface.scala:122)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.sql.hive.client.ClientInterface$class.getTable(ClientInterface.scala:122)\n\tat org.apache.spark.sql.hive.client.ClientWrapper.getTable(ClientWrapper.scala:60)\n\tat org.apache.spark.sql.hive.HiveMetastoreCatalog.lookupRelation(HiveMetastoreCatalog.scala:384)\n\tat org.apache.spark.sql.hive.HiveContext$$anon$2.org$apache$spark$sql$catalyst$analysis$OverrideCatalog$$super$lookupRelation(HiveContext.scala:458)\n\tat org.apache.spark.sql.catalyst.analysis.OverrideCatalog$class.lookupRelation(Catalog.scala:161)\n\tat org.apache.spark.sql.hive.HiveContext$$anon$2.lookupRelation(HiveContext.scala:458)\n\tat org.apache.spark.sql.SQLContext.table(SQLContext.scala:830)\n\tat org.apache.spark.sql.SQLContext.table(SQLContext.scala:826)</div>","startTime":1.465536402135E12,"submitTime":1.465536403918E12,"finishTime":1.465536402307E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"4de2da27-9d54-4ce2-816d-22f98e0ab59c"},{"version":"CommandV1","origId":124224,"guid":"a0562ee8-19ec-4e5f-b197-ba7e17206d85","subtype":"command","commandType":"auto","position":0.6875,"command":"%md\n###Scatter plot\nFour features were measured from each sample: the length and the width of the sepals and petals, in centimetres.\n\n<img src=\"https://upload.wikimedia.org/wikipedia/commons/5/56/Iris_dataset_scatterplot.svg\" width=\"600\">\n\nClustered using \n * k-means (left) and \n * true species in the data set (right). \n \n K-mean result by wikipedia, but it seems wrong\n \n","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536403927E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cff8f46e-c3cc-4067-a7c2-304c383f6eb3"},{"version":"CommandV1","origId":124225,"guid":"8cdba56b-c15d-4dc9-873f-21d19b45784b","subtype":"command","commandType":"auto","position":0.84375,"command":"%md\n##Section1. K-Mean cluster via Mllib kmeans","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536403935E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"e543556f-9225-42de-bcb1-79b1fa4b708e"},{"version":"CommandV1","origId":124226,"guid":"1e345e78-f31e-43ac-b495-60f4a738785e","subtype":"command","commandType":"auto","position":1.0,"command":"import org.apache.spark.mllib.clustering.{KMeansModel, KMeans}\nimport org.apache.spark.mllib.linalg.{Vector, Vectors}\nimport org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}\nimport org.apache.spark.{sql, SparkContext}\nimport org.apache.spark.mllib.clustering.{KMeans, KMeansModel}\nimport org.apache.spark.sql.{Row, SQLContext}\nimport org.apache.spark.sql.functions.monotonicallyIncreasingId","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import org.apache.spark.mllib.clustering.{KMeansModel, KMeans}\nimport org.apache.spark.mllib.linalg.{Vector, Vectors}\nimport org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}\nimport org.apache.spark.{sql, SparkContext}\nimport org.apache.spark.mllib.clustering.{KMeans, KMeansModel}\nimport org.apache.spark.sql.{Row, SQLContext}\nimport org.apache.spark.sql.functions.monotonicallyIncreasingId\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536402317E12,"submitTime":1.465536403942E12,"finishTime":1.465536402399E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"a3d63951-93d0-4b75-9952-448445fce5b1"},{"version":"CommandV1","origId":124227,"guid":"d3f57133-73c7-419f-bce3-9d9bfb2ca517","subtype":"command","commandType":"auto","position":2.0,"command":"def getKMeanCluster(fisheririsDF:sql.DataFrame, feaNames:Array[String], idcolName: String,\n nc:Int, sc: SparkContext, sqlContext: SQLContext ): sql.DataFrame ={\n val feasName0=feaNames\n val featureDF: sql.DataFrame = fisheririsDF.select(feasName0.head, feasName0.tail: _*).cache()\n val feasName=featureDF.columns.filter(x=>x!=idcolName)\n val feaRdd=featureDF.rdd.zipWithIndex()\n val outputID=feaRdd.map(v=>(v._2,v._1.getAs[Long](idcolName))).collect().sortBy(_._1).map(x=>x._2)\n\n def row2denseVector(row: Row, fname:Array[String]):Vector={\n val rowarray=Vectors.dense(Range(0,fname.length).toArray.map(v=>row.getAs[Double](fname(v))))\n rowarray\n }\n val dataArray=feaRdd.collect().sortBy(_._2).map(_._1).map(x=>row2denseVector(x,feasName))\n\n var mycluster=new KMeans()\n mycluster.setMaxIterations(10)\n mycluster.setK(nc)\n mycluster.setInitializationSteps(5)\n mycluster.setEpsilon(1e-4)\n\n val data=sc.parallelize(dataArray)\n\n val clusterModel:KMeansModel = mycluster.run(data)\n var labels: Array[(Int,Long)]=null\n if(outputID==null){\n labels = clusterModel.predict(data).collect().zipWithIndex.map(v=>(v._1,v._2.toLong))\n }else{\n labels = clusterModel.predict(data).collect().zip(outputID)\n }\n //disp labels\n import sqlContext.implicits._\n val labeldf = sc.makeRDD(labels).toDF(\"label\", \"id\").cache()\n\n labeldf.registerTempTable(\"clusterLabel\")\n fisheririsDF.registerTempTable(\"fisheriris\")\n\n val fisheririsCluster=sqlContext.sql(\n \"SELECT fisheriris.sepalLength, fisheriris.sepalWidth, \" +\n \"fisheriris.petalLength, fisheriris.petalWidth, \" +\n \"fisheriris.id, fisheriris.specie, clusterLabel.label FROM fisheriris, clusterLabel \" +\n \"WHERE fisheriris.id=clusterLabel.id\"\n ).orderBy(\"id\")\n\n \n //get unique class\n val groupCounts=fisheririsCluster.select(\"specie\",\"label\").groupBy(\"specie\",\"label\").count()\n groupCounts.show()\n\n fisheririsCluster\n\n }","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">getKMeanCluster: (fisheririsDF: org.apache.spark.sql.DataFrame, feaNames: Array[String], idcolName: String, nc: Int, sc: org.apache.spark.SparkContext, sqlContext: org.apache.spark.sql.SQLContext)org.apache.spark.sql.DataFrame\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536402404E12,"submitTime":1.465536403949E12,"finishTime":1.465536402885E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"a6d1fc39-5dfe-4c89-a38f-32b7edd97bd8"},{"version":"CommandV1","origId":124228,"guid":"b638574c-0919-4f65-888a-ba22ed989faa","subtype":"command","commandType":"auto","position":2.125,"command":"val feasName=Array(\"sepalLength\",\"sepalWidth\",\"petalLength\",\"petalWidth\",\"id\")\nval nc=3\nval fisherDFwithIndex=fisheririsDF.withColumn(\"id\",monotonicallyIncreasingId).cache()\n //cluster on raw feature\nval clusterOnRawFeature = getKMeanCluster(fisherDFwithIndex,feasName,\"id\",nc,sc,sqlContext)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------------+-----+-----+\n| specie|label|count|\n+------------+-----+-----+\n|'versicolor'| 1| 3|\n|'versicolor'| 2| 47|\n| 'setosa'| 0| 50|\n| 'virginica'| 1| 36|\n| 'virginica'| 2| 14|\n+------------+-----+-----+\n\nfeasName: Array[String] = Array(sepalLength, sepalWidth, petalLength, petalWidth, id)\nnc: Int = 3\nfisherDFwithIndex: org.apache.spark.sql.DataFrame = [sepalLength: double, sepalWidth: double, petalLength: double, petalWidth: double, specie: string, id: bigint]\nclusterOnRawFeature: org.apache.spark.sql.DataFrame = [sepalLength: double, sepalWidth: double, petalLength: double, petalWidth: double, id: bigint, specie: string, label: int]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536402896E12,"submitTime":1.465536403956E12,"finishTime":1.465536407867E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"52458722-697b-412b-94e7-92deab9cd087"},{"version":"CommandV1","origId":124229,"guid":"b6d6df27-b416-4da5-9a2c-e3bc3f92435d","subtype":"command","commandType":"auto","position":2.25,"command":"%md\nK-mean error ratio 11.33%","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536403964E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"0abea963-d19e-47cc-b2ed-fb4df45a20e6"},{"version":"CommandV1","origId":124230,"guid":"f5c0527e-f793-4e51-bd10-e88fa90e10a7","subtype":"command","commandType":"auto","position":2.5,"command":"clusterOnRawFeature.registerTempTable(\"kmeanTable\")\ndisplay(clusterOnRawFeature)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[5.1,3.5,1.4,0.2,0.0,"'setosa'",0.0],[4.9,3.0,1.4,0.2,1.0,"'setosa'",0.0],[4.7,3.2,1.3,0.2,2.0,"'setosa'",0.0],[4.6,3.1,1.5,0.2,3.0,"'setosa'",0.0],[5.0,3.6,1.4,0.2,4.0,"'setosa'",0.0],[5.4,3.9,1.7,0.4,5.0,"'setosa'",0.0],[4.6,3.4,1.4,0.3,6.0,"'setosa'",0.0],[5.0,3.4,1.5,0.2,7.0,"'setosa'",0.0],[4.4,2.9,1.4,0.2,8.0,"'setosa'",0.0],[4.9,3.1,1.5,0.1,9.0,"'setosa'",0.0],[5.4,3.7,1.5,0.2,10.0,"'setosa'",0.0],[4.8,3.4,1.6,0.2,11.0,"'setosa'",0.0],[4.8,3.0,1.4,0.1,12.0,"'setosa'",0.0],[4.3,3.0,1.1,0.1,13.0,"'setosa'",0.0],[5.8,4.0,1.2,0.2,14.0,"'setosa'",0.0],[5.7,4.4,1.5,0.4,15.0,"'setosa'",0.0],[5.4,3.9,1.3,0.4,16.0,"'setosa'",0.0],[5.1,3.5,1.4,0.3,17.0,"'setosa'",0.0],[5.7,3.8,1.7,0.3,18.0,"'setosa'",0.0],[5.1,3.8,1.5,0.3,19.0,"'setosa'",0.0],[5.4,3.4,1.7,0.2,20.0,"'setosa'",0.0],[5.1,3.7,1.5,0.4,21.0,"'setosa'",0.0],[4.6,3.6,1.0,0.2,22.0,"'setosa'",0.0],[5.1,3.3,1.7,0.5,23.0,"'setosa'",0.0],[4.8,3.4,1.9,0.2,24.0,"'setosa'",0.0],[5.0,3.0,1.6,0.2,25.0,"'setosa'",0.0],[5.0,3.4,1.6,0.4,26.0,"'setosa'",0.0],[5.2,3.5,1.5,0.2,27.0,"'setosa'",0.0],[5.2,3.4,1.4,0.2,28.0,"'setosa'",0.0],[4.7,3.2,1.6,0.2,29.0,"'setosa'",0.0],[4.8,3.1,1.6,0.2,30.0,"'setosa'",0.0],[5.4,3.4,1.5,0.4,31.0,"'setosa'",0.0],[5.2,4.1,1.5,0.1,32.0,"'setosa'",0.0],[5.5,4.2,1.4,0.2,33.0,"'setosa'",0.0],[4.9,3.1,1.5,0.2,34.0,"'setosa'",0.0],[5.0,3.2,1.2,0.2,35.0,"'setosa'",0.0],[5.5,3.5,1.3,0.2,36.0,"'setosa'",0.0],[4.9,3.6,1.4,0.1,37.0,"'setosa'",0.0],[4.4,3.0,1.3,0.2,38.0,"'setosa'",0.0],[5.1,3.4,1.5,0.2,39.0,"'setosa'",0.0],[5.0,3.5,1.3,0.3,40.0,"'setosa'",0.0],[4.5,2.3,1.3,0.3,41.0,"'setosa'",0.0],[4.4,3.2,1.3,0.2,42.0,"'setosa'",0.0],[5.0,3.5,1.6,0.6,43.0,"'setosa'",0.0],[5.1,3.8,1.9,0.4,44.0,"'setosa'",0.0],[4.8,3.0,1.4,0.3,45.0,"'setosa'",0.0],[5.1,3.8,1.6,0.2,46.0,"'setosa'",0.0],[4.6,3.2,1.4,0.2,47.0,"'setosa'",0.0],[5.3,3.7,1.5,0.2,48.0,"'setosa'",0.0],[5.0,3.3,1.4,0.2,49.0,"'setosa'",0.0],[7.0,3.2,4.7,1.4,50.0,"'versicolor'",1.0],[6.4,3.2,4.5,1.5,51.0,"'versicolor'",2.0],[6.9,3.1,4.9,1.5,52.0,"'versicolor'",1.0],[5.5,2.3,4.0,1.3,53.0,"'versicolor'",2.0],[6.5,2.8,4.6,1.5,54.0,"'versicolor'",2.0],[5.7,2.8,4.5,1.3,55.0,"'versicolor'",2.0],[6.3,3.3,4.7,1.6,56.0,"'versicolor'",2.0],[4.9,2.4,3.3,1.0,57.0,"'versicolor'",2.0],[6.6,2.9,4.6,1.3,58.0,"'versicolor'",2.0],[5.2,2.7,3.9,1.4,59.0,"'versicolor'",2.0],[5.0,2.0,3.5,1.0,60.0,"'versicolor'",2.0],[5.9,3.0,4.2,1.5,61.0,"'versicolor'",2.0],[6.0,2.2,4.0,1.0,62.0,"'versicolor'",2.0],[6.1,2.9,4.7,1.4,63.0,"'versicolor'",2.0],[5.6,2.9,3.6,1.3,64.0,"'versicolor'",2.0],[6.7,3.1,4.4,1.4,65.0,"'versicolor'",2.0],[5.6,3.0,4.5,1.5,66.0,"'versicolor'",2.0],[5.8,2.7,4.1,1.0,67.0,"'versicolor'",2.0],[6.2,2.2,4.5,1.5,68.0,"'versicolor'",2.0],[5.6,2.5,3.9,1.1,69.0,"'versicolor'",2.0],[5.9,3.2,4.8,1.8,70.0,"'versicolor'",2.0],[6.1,2.8,4.0,1.3,71.0,"'versicolor'",2.0],[6.3,2.5,4.9,1.5,72.0,"'versicolor'",2.0],[6.1,2.8,4.7,1.2,73.0,"'versicolor'",2.0],[6.4,2.9,4.3,1.3,74.0,"'versicolor'",2.0],[6.6,3.0,4.4,1.4,75.0,"'versicolor'",2.0],[6.8,2.8,4.8,1.4,76.0,"'versicolor'",2.0],[6.7,3.0,5.0,1.7,8.589934592E9,"'versicolor'",1.0],[6.0,2.9,4.5,1.5,8.589934593E9,"'versicolor'",2.0],[5.7,2.6,3.5,1.0,8.589934594E9,"'versicolor'",2.0],[5.5,2.4,3.8,1.1,8.589934595E9,"'versicolor'",2.0],[5.5,2.4,3.7,1.0,8.589934596E9,"'versicolor'",2.0],[5.8,2.7,3.9,1.2,8.589934597E9,"'versicolor'",2.0],[6.0,2.7,5.1,1.6,8.589934598E9,"'versicolor'",2.0],[5.4,3.0,4.5,1.5,8.589934599E9,"'versicolor'",2.0],[6.0,3.4,4.5,1.6,8.5899346E9,"'versicolor'",2.0],[6.7,3.1,4.7,1.5,8.589934601E9,"'versicolor'",2.0],[6.3,2.3,4.4,1.3,8.589934602E9,"'versicolor'",2.0],[5.6,3.0,4.1,1.3,8.589934603E9,"'versicolor'",2.0],[5.5,2.5,4.0,1.3,8.589934604E9,"'versicolor'",2.0],[5.5,2.6,4.4,1.2,8.589934605E9,"'versicolor'",2.0],[6.1,3.0,4.6,1.4,8.589934606E9,"'versicolor'",2.0],[5.8,2.6,4.0,1.2,8.589934607E9,"'versicolor'",2.0],[5.0,2.3,3.3,1.0,8.589934608E9,"'versicolor'",2.0],[5.6,2.7,4.2,1.3,8.589934609E9,"'versicolor'",2.0],[5.7,3.0,4.2,1.2,8.58993461E9,"'versicolor'",2.0],[5.7,2.9,4.2,1.3,8.589934611E9,"'versicolor'",2.0],[6.2,2.9,4.3,1.3,8.589934612E9,"'versicolor'",2.0],[5.1,2.5,3.0,1.1,8.589934613E9,"'versicolor'",2.0],[5.7,2.8,4.1,1.3,8.589934614E9,"'versicolor'",2.0],[6.3,3.3,6.0,2.5,8.589934615E9,"'virginica'",1.0],[5.8,2.7,5.1,1.9,8.589934616E9,"'virginica'",2.0],[7.1,3.0,5.9,2.1,8.589934617E9,"'virginica'",1.0],[6.3,2.9,5.6,1.8,8.589934618E9,"'virginica'",1.0],[6.5,3.0,5.8,2.2,8.589934619E9,"'virginica'",1.0],[7.6,3.0,6.6,2.1,8.58993462E9,"'virginica'",1.0],[4.9,2.5,4.5,1.7,8.589934621E9,"'virginica'",2.0],[7.3,2.9,6.3,1.8,8.589934622E9,"'virginica'",1.0],[6.7,2.5,5.8,1.8,8.589934623E9,"'virginica'",1.0],[7.2,3.6,6.1,2.5,8.589934624E9,"'virginica'",1.0],[6.5,3.2,5.1,2.0,8.589934625E9,"'virginica'",1.0],[6.4,2.7,5.3,1.9,8.589934626E9,"'virginica'",1.0],[6.8,3.0,5.5,2.1,8.589934627E9,"'virginica'",1.0],[5.7,2.5,5.0,2.0,8.589934628E9,"'virginica'",2.0],[5.8,2.8,5.1,2.4,8.589934629E9,"'virginica'",2.0],[6.4,3.2,5.3,2.3,8.58993463E9,"'virginica'",1.0],[6.5,3.0,5.5,1.8,8.589934631E9,"'virginica'",1.0],[7.7,3.8,6.7,2.2,8.589934632E9,"'virginica'",1.0],[7.7,2.6,6.9,2.3,8.589934633E9,"'virginica'",1.0],[6.0,2.2,5.0,1.5,8.589934634E9,"'virginica'",2.0],[6.9,3.2,5.7,2.3,8.589934635E9,"'virginica'",1.0],[5.6,2.8,4.9,2.0,8.589934636E9,"'virginica'",2.0],[7.7,2.8,6.7,2.0,8.589934637E9,"'virginica'",1.0],[6.3,2.7,4.9,1.8,8.589934638E9,"'virginica'",2.0],[6.7,3.3,5.7,2.1,8.589934639E9,"'virginica'",1.0],[7.2,3.2,6.0,1.8,8.58993464E9,"'virginica'",1.0],[6.2,2.8,4.8,1.8,8.589934641E9,"'virginica'",2.0],[6.1,3.0,4.9,1.8,8.589934642E9,"'virginica'",2.0],[6.4,2.8,5.6,2.1,8.589934643E9,"'virginica'",1.0],[7.2,3.0,5.8,1.6,8.589934644E9,"'virginica'",1.0],[7.4,2.8,6.1,1.9,8.589934645E9,"'virginica'",1.0],[7.9,3.8,6.4,2.0,8.589934646E9,"'virginica'",1.0],[6.4,2.8,5.6,2.2,8.589934647E9,"'virginica'",1.0],[6.3,2.8,5.1,1.5,8.589934648E9,"'virginica'",2.0],[6.1,2.6,5.6,1.4,8.589934649E9,"'virginica'",1.0],[7.7,3.0,6.1,2.3,8.58993465E9,"'virginica'",1.0],[6.3,3.4,5.6,2.4,8.589934651E9,"'virginica'",1.0],[6.4,3.1,5.5,1.8,8.589934652E9,"'virginica'",1.0],[6.0,3.0,4.8,1.8,8.589934653E9,"'virginica'",2.0],[6.9,3.1,5.4,2.1,8.589934654E9,"'virginica'",1.0],[6.7,3.1,5.6,2.4,8.589934655E9,"'virginica'",1.0],[6.9,3.1,5.1,2.3,8.589934656E9,"'virginica'",1.0],[5.8,2.7,5.1,1.9,8.589934657E9,"'virginica'",2.0],[6.8,3.2,5.9,2.3,8.589934658E9,"'virginica'",1.0],[6.7,3.3,5.7,2.5,8.589934659E9,"'virginica'",1.0],[6.7,3.0,5.2,2.3,8.58993466E9,"'virginica'",1.0],[6.3,2.5,5.0,1.9,8.589934661E9,"'virginica'",2.0],[6.5,3.0,5.2,2.0,8.589934662E9,"'virginica'",1.0],[6.2,3.4,5.4,2.3,8.589934663E9,"'virginica'",1.0],[5.9,3.0,5.1,1.8,8.589934664E9,"'virginica'",2.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"sepalLength","type":"\"double\""},{"name":"sepalWidth","type":"\"double\""},{"name":"petalLength","type":"\"double\""},{"name":"petalWidth","type":"\"double\""},{"name":"id","type":"\"long\""},{"name":"specie","type":"\"string\""},{"name":"label","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.465536407872E12,"submitTime":1.46553640397E12,"finishTime":1.465536408078E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c5e3ce11-30c4-475f-ad55-3679fa9cd05f"},{"version":"CommandV1","origId":124231,"guid":"18b71f2f-b94d-495e-ace8-f7cc69c9fc37","subtype":"command","commandType":"auto","position":2.546875,"command":"%py\n#do plot with true label\nimport numpy as np\nfrom mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt\n\ndf = sqlContext.table(\"fisheriris\")\ny=df.collect()\nfea1 = np.asarray(map(lambda x: x[0], y))\nfea2 = np.asarray(map(lambda x: x[1], y))\nfea3 = np.asarray(map(lambda x: x[2], y))\ntl= np.arange(150)\ntl[range(0,50)]=0\ntl[range(50,100)]=1\ntl[range(100,150)]=2\n\nfig1 = plt.figure(figsize=(20,10))\nax = fig1.add_subplot(121, projection='3d')\nfigpars=[('r', 'o'), ('m', 'x'), ('b','^')] #color and marker\nsymbol=['setosa','versicolor','virginica']\nfor i in range(0,3):\n xs = fea1[tl==i]\n ys = fea2[tl==i]\n zs = fea3[tl==i]\n ax.scatter(xs, ys, zs, c=figpars[i][0], marker=figpars[i][1])\n plt.legend(symbol)\n plt.hold(True)\n\nplt.hold(False)\nfor ii in xrange(60,160,30):\n ax.view_init(elev=10., azim=ii)\nplt.axis('tight')\nplt.title('Iris flower class')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\n\n#do plot with kmean\ndfkm = sqlContext.table(\"kmeanTable\")\nyy=dfkm.collect()\nfea1 = np.asarray(map(lambda x: x[0], yy))\nfea2 = np.asarray(map(lambda x: x[1], yy))\nfea3 = np.asarray(map(lambda x: x[2], yy))\ntl = np.asarray(map(lambda x: x[6], yy))\n\nax = fig1.add_subplot(122, projection='3d')\nfigpars=[('r', 'o'), ('m', 'x'), ('b','^')] #color and marker\nsymbol=['cluster_0','cluster_1','cluster_2']\nli=[0,1,2]#This is used just to match the marker of two figure below more comparable\nfor i in range(0,3):\n xs = fea1[tl==li[i]]\n ys = fea2[tl==li[i]]\n zs = fea3[tl==li[i]]\n ax.scatter(xs, ys, zs, c=figpars[i][0], marker=figpars[i][1])\n plt.legend(symbol)\n plt.hold(True)\n\nplt.hold(False)\nfor ii in xrange(60,160,30):\n ax.view_init(elev=10., azim=ii)\nplt.axis('tight')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nplt.title('Kmean-cluster')","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">5</span><span class=\"ansired\">]: </span><matplotlib.text.Text at 0x7f9cce34f090>\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536408084E12,"submitTime":1.465536403978E12,"finishTime":1.465536408889E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"cbd9335e-4308-4988-b770-20b956e1e7d4"},{"version":"CommandV1","origId":124232,"guid":"5ca4c974-2b03-4f11-838c-a1b34bab769b","subtype":"command","commandType":"auto","position":2.59375,"command":"%py\ndisplay(fig1)","commandVersion":0,"state":"finished","results":{"type":"image","data":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAB9AAAAPoCAYAAACGXmWqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xm81mP++PHXdZ+WU9oUpamh0GIZqVQqUkIxJiZLvhiVbcgMwm+YMUM1iC8z0pD1i2QdMpYxJpKSkFNUhkJDZUmbFq1O51y/Pz73Oc5yn6WcFF7Px+N6NOdzrZ/7Ho/HfX3en+u6QowRSZIkSZIkSZIkSZJ+7FLbewCSJEmSJEmSJEmSJO0IDKBLkiRJkiRJkiRJkoQBdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAFQbXsPQJIkSdKOIYSwO7DL9h6HVIHlMcZF23sQkiRJUlHOp7QDcc4kfUsG0CVJkiQRQtg9lUq9n5+fn729xyKVJ5VKbQwhtPGBkCRJknYUzqe0I3HOJH17BtAlSZIkAeySn5+f/eCDD7LPPvts77FIGc2dO5fTTz89m2Rljw+DJEmStKNwPqUdgnMmqWoYQJckSZJUaJ999qFDhw7bexiSJEmS9L3jfEqSfhhS23sAkiRJkiRJkiRJkiTtCAygS5IkSZIkSZIkSZKEAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiTpO7Jw4UJSqRRnnnnm9h6KJGVkAF2SJEmStrGxY8eSSqV44IEHtvdQJEmSJOlHZ9iwYaRSKV555ZXtPZSM8vPzufnmm2nXrh21a9emUaNG/PznP+f111/f3kOTfpQMoEuSJEnSdyCEsL2HIEmSJEk/SiGEHXpONmDAAC699FJyc3P57W9/S//+/Zk6dSo9evTg2Wef3d7Dk350qm3vAUiSJEnSD12McXsPQZIkSZJ+tHbkOdkjjzzC+PHjOeSQQ5g4cSI1atQA4LzzzqN79+6cc845zJ8/nzp16mznkUo/Hq5AlyRJkvSd+eCDDxgyZAhdOnSg37HH7pBv0v/jH/+gV69eNG3alOzsbJo2bcqhhx7KmDFjipVbuXIlv//979l3332pXbs2DRo04IgjjuDFF18sVq5Xr16FZ/sNGjSIVCpFKpUiKyuLRYsWFZZbvXo1V1xxBW3atKFWrVo0bNiQPn36MHHixIzjvPfee+nWrRuNGzemVq1aNGvWjCOPPJLHHnusWLnJkydz7rnnst9++1G/fn1q167N/vvvz7Bhw9i4cWNVfGSSJEmSvgPLly/n6quvplvnzvTu2ZO77rqL3Nzc7T2sUnJychgwYADNmzcnOzubn/zkJ/Tp04fHH3+83Ho9e/YklcoctirrWKxZs2YxYMAAWrRoQXZ2Nrvssgvt2rXj4osvJi8vD4CWLVsyYsSIYn0UzMmK2rBhAyNHjqR9+/bUqVOHunXr0q1bNx599NFS45kyZQqpVIoRI0Ywffp0+vbtS8OGDUvN8ypjzJgxhBC49tprC4PnAB07dmTAgAEsW7aMJ554YovalPTtuAJdkiRJ0laJMbJx40Zq1qxZ5kOOoqZPn07vXr2om5tL382bmTtnDv2ee44//elPhQ8ztrfbb7+dCy64gKZNm3Lcccexyy67sHTpUubMmcPYsWMZMmQIAIsWLeKwww5j0aJF9OjRg2OOOYa1a9fyz3/+k759+3LXXXdx1llnATB48GB23nlnnn76aY4//ngOPPBAINlCsEGDBgCsWrWKrl278sEHH9C5c2dOOOEEli9fzt///nf69OnDbbfdxnnnnVc4zssvv5wbb7yRPffckwEDBlC/fn0WL15MTk4O48ePZ8CAAYVlb7jhBt5//326devGsccey4YNG5g2bRojRoxg8uTJTJo0qVLfnyRJkqSqtWnTJrKysqhWreJQzeLFi+nepQtLP/uMX+TnsyoEzpsyhWeffpqnnnmmVDB4e7n77rsZMmQI1apVo1+/frRq1YqlS5eSk5PD7bffzkknnVRm3Yq2WS+ZN3v2bLp27UpWVhb9+vWjZcuWrFmzhvnz53PHHXdw3XXXUbt2bYYOHcpTTz3FlClTGDRoEC1atCjV3urVq+nVqxezZ8+mY8eOnHXWWeTn5zNhwgROPfVU3nvvvYzz1mnTpnHttdfSo0cPzj77bJYuXVosCF6RTZs28cYbb1C7dm0OOeSQUvlHH30048aN4+WXX2bQoEGVblfStxRjNJlMJpPJZDKZTD/yBHQA4syZM2NlPPzww3Hf1q0jEOvXqRMvueSSuG7dunLrdDnooHhQKhXXQozpNAxiCCF+9NFHFfa5YcOG+N5778WlS5dWaoxbo0OHDjE7OzsuX768VN6KFSsK//dhhx0Ws7Ky4vjx44uVWb16dTzwwANj7dq145IlSwqv33///TGVSsWxY8dm7Pecc86JIYT4m9/8ptj1Dz74INarVy/WrFkzLliwoPB6w4YNY/PmzePGjRvLHWeMMX788ccZ+7zyyitjKpWKjz32WMb8HdHMmTMjEIEOcQf478ZkMplMJpPJZIpxy+dTr776auzRvXsEYvVq1eL/nHJK/PTTT8utc8EFF8RdqlWLC4rMp55LfhvHJ554osI+N2/eHN9///24aNGiSo1xa7z33nuxevXqsVGjRnHevHml8gvuccGCBTGEEAcPHlwsv2fPnjGVSmVsO9Oc6pJLLompVCo+88wzpcqvWrWq2N/Dhg2LqVQqTpkyJWP7AwcOjKlUKt58883Frm/atCn27ds3ZmVlxVmzZhVenzx5cgwhxFQqFe++++6MbVbGu+++G0MI8YADDsiYP2PGjBhCiAcffHCl2nPOZDJVTXKZgSRJkqQtMnbsWE499VT2/vBD7gfOW7uWO0aN4sRf/pIYM58rt2TJEqbPmMEl+fnsVOT6/wOqQ7lbuccYGTlyJD9p0oR9992X3Zo04YRf/pJly5ZV4V0lQghUq1Yt4+qNhg0bAjBnzhxeeeUVTjzxRPr371+sTL169Rg+fDgbN25k/PjxleozNzeXhx56iLp163LNNdcUy2vVqhUXXnghubm5jBs3rtg4a9SokXF1RsE4CxSsrihp6NChxBh54YUXKjVOSZIkSd/ejBkz6N2rF+tff507gD9v3syUxx+nR7durFmzpsx6z4wfz682b2aPIteOAdpnZfH000+X2+fjjz/O3i1a0KZNG3bffXe6de7MnDlzquR+ihozZgx5eXlcddVVtGnTplR+s2bNqrS/gvlQrVq1SuXVr1+/0u18+eWXPPTQQ3Tq1ImLL764WF6NGjW44YYbyM/P5+GHHy5V98ADD+Tss8/ewpF/Y/Xq1eWOt+D6qlWrtroPSVvOLdwlSZIkVVpeXh5XX3klA4BHYqQgfNs5P58TXniBN954g65du1ZpnzfddBN/+MMfuBDoD8yNkauffZZjjjqK6TNnVun246eddhqXXXYZ++23H6eccgo9evSge/fu7LLLLoVlXn/9dSB5gDF8+PBSbSxdupQYI/PmzatUn++//z4bNmzg0EMPzfjQpHfv3lx77bW89dZbxcZ56623st9++3HyySfTo0cPunbtSr169UrVX79+PaNGjeKpp57igw8+4Kuvvip80SGEwGeffVapcUqSJEn69q675hr2zM/n1fx8aqavnZiXR9tPPmHs2LH89re/rdL+Jk6cyIABA+gXI3cAq4Br33qL3j178u68eTRu3LjK+po+fToAffv2rbI2y3PKKadwyy23cNxxx3HSSSfRu3dvunfvzp577rlF7eTk5BSel55pjvf1118DZJzjde7ceStGLmlHZwBdkiRJUqV9/vnnLPzsM/4GFF37fDywUyrFtGnTMgbQmzRpQpeDDuKvb71FvyKr0G8EcoFf/OIXGfvLzc3lpuuv53zglvS1w4C2eXn0mjWLl156iSOPPLKK7i5Zlb3rrrsyZswYRo8ezahRowDo1asXN954I+3bt2fFihUAvPjii7z44osZ2wkhsG7dukr1WbDiYLfddsuY37Rp02LlAEaNGsVee+3Ffffdx/XXX8/IkSOpVq0aP//5z/nrX/9Ky5YtAdi8eTO9evUiJyeHn/3sZ5xyyinsuuuuVK9eHYBhw4axadOmSo1TkiRJ0rf32tSp/DovrzB4DrAX0DUEpk2bVmYAvd8JJ/DAHXdwUV5e4Sr0fwFv5+Vx5XHHldnf/44cSadUiifz8gq3JO6Vl0eL1au55557+MMf/lAFd5UoWCVd1SvNy3LQQQfx6quvcu211/LEE08wbtw4Yoy0bduWYcOGcfLJJ1eqnYI5Xk5ODjk5ORnLlDXHK2seV1kFL1EXne8VVXC9QYMG36ofSVvGLdwlSZIkVVq9evXISqVYWOL6EmBDjKW2Dy/qlltvZW7NmuxdrRqDgS5ZWQwD/vjHPxYGfEtavHgxS7/8kpLh9cOAullZzJo1a6vvpSynn346r732GitWrOC5557j7LPPZvLkyfTp04cVK1YUPuC45ZZbyMvLKzPdc889leqvoL0vvvgiY/7ixYuLlYPk4c2FF17I22+/zZIlS3jyySfp378/Tz/9NEcffTSbN28G4OmnnyYnJ4fBgwcze/Zs7rjjDv785z9z1VVXce655271ZyRJkiRp6zRs2JAFJa7lA4tSqXLnU1deeSX1fvIT9svK4n+AviFwLHDsMcdw/PHHl1lv9ttv8/MiwXOAxsDBwOzZs7f2NjIqCPJu7S5XBbuL5efnl8orawvzLl268Mwzz7By5UqmTZvGVVddxZIlSzj11FOZPHlypfotmGsNHTq03DnexIkTS9XNdKzWlthrr73Iysrio48+ynjfH374IQCtW7f+Vv1I2jIG0CVJkiRVWv369fnl8cdzbVYWBRuKfwmcHwK1a9XihBNOKLNuly5deGvWLPqfey7vtW/PbkcfzTPPPMOIESPKrNOoUSNqVq9Oycc6HwFf5eXRvHnzb3tLZapXrx59+/blzjvvZNCgQaxYsYKpU6dy8MEHAzB16tRKt5WVlUWMsXBbwKLatGlD7dq1mT17dsYzDydNmgRAx44dM7a9yy67cPzxx/Poo49y+OGH8+GHH/Kf//wHgPnz5xNCKHVWO1Dph0mSJEmSqs7As8/mkVSKJ4EIbAL+CCzcvJmBAweWWa9p06a8+dZbXHrllSzs1IncHj248667ePKpp8jKyiqzXrPmzZldIsj7NfBeKlXlK8UL5koTJkzYqvo777wzAJ988kmpvLJWhheoXr06Bx98MMOGDeOWW24hPz+fp556qjC/4DPKNCfr3LkzqVRqi+Z4VaVmzZp069aN9evXZ+z/X//6FyEEevfu/Z2PTfoxM4AuSZIkaYv87bbbaLT33nQEWlavzk9SKV6oUYNHHnss4xneRbVu3ZrbbruN6W+9xdPPPlvm1u0FdtppJ351xhlcl5XFk0AeMBf4VSpF40aNyl1psTXKCiovWbIEgOzsbDp27Mihhx7Kk08+yX333Zex/H/+8x+WLVtW+HejRo2AzA+CqlevzmmnncaaNWv405/+VCzvv//9L6NHj6ZGjRqcfvrpQHL+3muvvVaqndzc3MKtB7OzswFo0aIFMUZefvnlYmU/+ugjrrjiim+9WkKSJEnSlhk6dChHH3MMJwDNqlWjSVYWI4GRI0fSpUuXcuvusssuDB8+nNfefJOXJk/mnHPOKTyeqSy/vuACnoyRvwLrgS+As4Fl+fmcddZZVXNTaeeffz5ZWVmMGDGC999/v1R+RSvTu3TpQoyRu+++u9j1l156iUcffbRU+ddff52NGzeWul6wu1fBvAiSOVmMMeOcbNddd+W0005jxowZXHPNNRlXgn/00UcsWLCg3PFvrfPPP58YI3/84x+LHbGVk5PD3//+dxo3bpzxpWhJ245noEuSJEnaIrvtthtvzZnD008/zYwZM9htt9049dRTadKkyTbp768338yiBQs44aWXSJFsb9i0USOe/ec/qVWrVpX29ctf/pI6depw8MEHFwafp06dSk5ODgcddBBHHHEEAA8//DC9e/fm7LPPZvTo0XTp0oUGDRrw6aefMmfOHN59911ef/11dt11VwC6du1K7dq1GTVqFMuXLy/8rC688ELq1q3L9ddfz9SpU7n11lt588036dWrF8uWLePxxx9n7dq13HbbbeyxR3LS4YYNGzjkkEPYe++96dixI3vssQcbN27kxRdfZN68efziF7+gbdu2QHK2/N57783NN9/MO++8Q/v27Vm4cCHPPfccxx57bMaHUJIkSZK2nRo1avDUM88wefJkXnzxRWrVqsXJJ59MmzZttkl/55xzDu/+5z9cdtttXBYjEahVsyZj77mH/fbbr0r72meffRgzZgznn38+Bx54IP369aNVq1YsX76cnJwcGjRowEsvvVRm/cGDB3PTTTcxcuRIZs2axb777ssHH3zAv//9b/r3788TTzxRrPz//u//MmnSJA499FBatmxJnTp1ePfdd3n++edp2LBhsWOrevXqRSqV4oorrmDOnDmFq92vvPJKAG699Vbmz5/P1Vdfzbhx4zjkkENo0qQJn3/+OXPnzmXGjBk88sgjtGjRoko/M4BTTjmFJ598kvHjx9O+fXt+8YtfsHz5cv7+97+Tn5/P3XffTZ06daq8X0nliDGaTCaTyWQymUymH3kCOgBx5syZcUf15ptvxjFjxsQnn3wybty4cZv0ceedd8b+/fvHvfbaK+60006xUaNGsUOHDvGmm26Ka9euLVZ27dq1ceTIkfGggw6KdevWjbVr14577rlnPPbYY+M999wT169fX6z8hAkTYrdu3WLdunVjKpWKqVQqLly4sDB/9erV8YorroitW7eO2dnZceedd459+vSJEydOLNZObm5uvPHGG+MxxxwT99hjj1irVq3YuHHj2LVr13jXXXfF3NzcYuU//fTTePrpp8fmzZvH2rVrx/333z/edNNNcfPmzTGVSsXDDz+8ij/FbWfmzJmRZKfLDnEH+O/GZDKZTCaTyWSK8fsxn5o/f36888474wMPPBC//PLLbdrXG2+8EU888cTYpEmTWLNmzdisWbN49NFHx/Hjx8cYY1ywYEFMpVLxzDPPLFV37ty58dhjj4316tWLdevWjb169YpTp06N999/f0ylUnHs2LGFZV988cV45plnxv322y82aNAg1qlTJ7Zt2zZefPHFcdGiRaXafuihh2L79u1j7dq1YyqVillZWcXyc3Nz42233Ra7d+8eGzRoELOzs+Mee+wRjzjiiDh69Ohin9vkyZNjKpWKI0aMqJLPLC8vL44aNSoecMABsXbt2rFhw4bx2GOPjW+88cYWteOcyWSqmhRijNsrdi9JkiRpBxFC6ADMnDlzJh06dNjew5EyeuuttwrOgu8YY3xre49HkiRJAudT2nE4Z5KqhmegS5IkSZIkSZIkSZKEAXRJkiRJkiRJkiRJkgCotr0HIEmSJEmSJEmSJP0QjBo1itWrV1dYrmfPnhx22GHfwYgkbSkD6JIkSZIkSZIkSVIVuOWWW1i0aFGF5UIIBtClHZQBdEmSJEmSJEmSJKkKfPzxx9t7CJK+Jc9AlyRJkiRJkiRJkiQJA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5Kk70gIYXIIIX8btd04hHBfCGFRCGFzCCEvhFAvhDAohJAfQjhjW/T7fRdC2CP9+dy7vcciSZIkSVsrhHB/em6z+/YeiyRJ+v4zgC5JkrZI+qFE3lZUjcA2CaAD9wO/AqYAfwaGARvTfcZt1KckSZIkVUp586gQQqsQwsfpMtd812P7gfjO537p72vSd9mnJEn6blTb3gOQJEk/Gr8Cald1oyGE6sBRwIsxxl+VyKvq7iRJkiSpyoQQOgHPATsDF8QYb9/OQ5IkSfrRM4AuSZK+EzHGT7dR001JdtVZvI3alyRJkqQqF0I4ChhP8ox2QIzxye08JEmSJOEW7pIkqQoUPUs7hNAmhPBECGFp+izyHukyGc9ADyGcGUJ4LV1+QwjhsxDCiyGEAZXo92NgAclWfQXnnVfqTO8QQscQwpMhhCUhhI0hhAUhhDEhhKYlyl2XbrN3ievXpq9/mKHtL0IICzJc7xNC+FcIYVm6z/khhP8NIdTPUHZBCOGj9Fnut6T//jqEcFVF95au3ymE8FgI4dN0X5+HECaEEE6qRN1WIYTrQwg56e+l4PO5K4Tw0zLqVOp7DCEcmB7XgnS7y0MIs0MIo0IIWZW5N0mSJOn7LoRwKvAskAv0yRQ8L9giPITQOD3XWhxCWJf+3X1YukztEMJfQggL07+v3w0hnFhOv/8TQng5hLAy/bv9vRDClSGEGhnKHh9CGBdCeD+EsDadZoYQLgohlHquXPQc8hDCr0MIc9J9fJGeS5Sa91Tic6oVQrg8PTdZE0L4Kj3mW0IIu1ZQ97D0eDLOoQrmXCWu1QghDA0hvBVC+DKEsD6EsCiE8EzBnDCEMDAkc9sI9CwyDy3VVwihS0jmx4tDCJvSbd1Rct6ZLjs5JHPoGiGEESGED9J1KpzfSpKkqmUAXZIkVaVWwBtAM2AccA/wVTqv1Jl0IYQb0mUaA48BfwH+DTQBTqhEfzcDo4AAzCI5+3wY8FR5lUIIxwGvAUcDL6T7fR84D8gJIbQoUvyl9L+Hl2jm8PT97BlC2L1I2/ul72diiT6vBp4HDgL+CdwCfAhcBrwaQqhbov0I1AQmAT8n+VxGk7wwUK4Qwjnp++sHTANuSvfZGDi/ovpAf+BcYBHwcLrfucBZwJshhJ+U6K9S32MIoR3wevp+XkuXewT4nOSzr1mJsUkCFi5cSCqV4swzz/zO+hw7diypVIoHHnjgO+tTkqQfohDCxSTzpWXAYTHGV8op3oDkN307kt/OjwMdgedDCB2BySS/r58B7ieZiz0aQuicod97gYeAlul2bgVWAH9Ot1fyhdaRQHuSOd5o4AGgDsk8LNMPgoI5343A9SRztFuBT4GzqWCelmG8DUjmDyNJjgP7P2AMydxkMLDPlrRXxnhLGkcyT8kCxpLMNycB+wJ90mUK5p6BZH42rEiaXGT8ZwKvkhw59hLc4RHsAAAgAElEQVTJ55ZDMq+aEUJoXsZ4xpN8XlPT/f9nq+5OkiRtNbdwlyRJVakbcF2M8U+VLH828BmwX4xxU9GMEELDiirHGEeHEPYALgZmxRhHVFQnhLATcC/Jw46eMcbpRfIuJ3k4cyffPByZBmwCegNXpsvVIwmEvwgcmc67L12+N8mDj5eLtNsLuJrkAcjPY4xri+SdQfKgazhwSYnh7kbysOTQGOOGiu4t3d4+wG3AaqB7jPH9EvnNKtHMA8BfY4y5Jer2ITmf8Y/AkCJZlf0ezwBqAMfHGJ8tUa5+jHF9JcYmKS2EQAjhO+9TkiRtvRDCSOBykhd4+8QYF1VQpR1wR4zxgiJtTCT5zT6JZN5xSIzx63TeWJI5zOUUf5l1EDAI+DtwRkH5dN5VJPOVC0gC5QWOiTF+nOEexgGnhhBGxxjfLJkNdAb2jzF+li6fSo+zRwihU4wxp4J7LjAG+BlwW4zxtyXGsBNJkLvKpOd5JwIzYoyZXkDYGSDGOBuYHUIYBizINA8NIbQCbgfmk7wksbRIXi+SueRokpeXi1UFfkoyt1pZFfcl7YgWLlxIy5YtGTRoEPfe6yYLknY8rkCXJElVaQlQYRC7iAh8TYY3/2OMX1bVoEo4HtgZeKRo8DztLyQrCI4I6a3KY4wbSVY9dCiySrwnye+ovwBfkgTNCxyR/velItcuJLnHXxcNnqfbf4BkBcOpZYz3ssoGz9OGkDxIGlEyeJ7u77OKGogxLi4ZPE9fnwC8R7KColgWlfseC/JL3U+McXVF45L0jWbNmjF37lyuu+667T0USZK0ZS4n+e18dCWC5wDrgd+VuPYwsJlkNfhFRYPhMcbXgY+BA0vUuSjd77lFy6ddQzKvOa3oxUzB87SCXcBKzgsg+c0/vOi8I8aYT/LCcQA6ldFmMent2U8GFgP/r1QnMa6LMa6pTFtbIJ9kjJsyZW5hQHsIyeK1oUWD5+l2XibZMeAXIYQ6JbsB/mTwXKp6w4YNI5VK8cor5W36sX3MmzePq6++muOPP5499tiDVCpFVlYW+fmlTkKU9B1xBbokSapKszMFXsvxEPAb4N0Qwt+BV4DXt8GDkKLakzyUmFwyI8a4OYTwCvCrdLlP0lmTgMNIAufPkmzfvik93pfTfxesrDgUeD/G+EWRpg8mOdtwQBmrN2sAu4YQdi7xoGRjjPGdLby/Lul//72F9YoJIZwODCRZ8bIzxVd3lHygVNnv8VGSB3dPhxAeJ3nJYFqM8SMkbZFq1arRunXr7T0MSZK05f4N9AUeCSH0rcSLpB/EGNcVvRBjzA8hLAFqxxgXZqjzOckqcCA5Rxw4gGTL+KEZ5iQFQeO2xS4mu0n9juToqz2BnYoOg2S7+ExmZrhWMLfauUj7A4EWJcpNjjFOIQm0p4BX0i81b3MxxrUhhGeBY0MIb5Nspf4qMH0LX2qGZA4I0CuE0CVDfmOS+2sFvF0ir+SqfklVYHvs4FVZEyZM4M9//jPVqlWjVatWZGdns2lTxnd5JH1HXIEuSZKq0hcVFynmYmAosBa4guSM8OUhhH+EEFpW9eDS6qf/LWusi0uUgyTQG/hmpfnhJAHiTem8JiGEfUm2da9P8dXnAI2A6sBVZaR9SB5AlVx9sJQt1yD9b4UrzcsSQig403Afkgd8N/HNmX4LSQL+RVXqe4wxzgAOIfl8TiTZun5+COG9EMLJWztefX+s/2A9a94s/l7F5q82s/zp5dtpRMVNnz6dVCrFCSecUGaZ1q1bU6tWLVatWlV4bcKECRxzzDHsuuuuZGdns/fee/O73/2O1atLPw9v0aIFe+65J2vWrOGiiy6iRYsW1KhRgxEjks071qxZw/Dhw9l///2pV68edevWpWXLlpx88sm8/fY3z1bLOwN9w4YN3HDDDXTq1KmwjX333ZeLLrqIZcuWFSu7ePFihgwZQsuWLalZsyaNGzemf//+zJgxY4s+u5kzZ9K/f3+aNGlCdnY2LVq0YMiQISxevLhU2UGDBpFKpViwYAGjRo3igAMOoHbt2hx++OFb1KckSd9Tx5GsPu4CTKrE0VVlBdg3V5BXdOHUziTzmV3JPB/5E9CUIgHyEEJ9YAbJ6u/1JOeBX0MyJyhYgV5zC8a8Of1v0RdzB2UYy2HpvG89r9lKJ5Mcr1Ur/e8kYEUI4f4Qwi5b0E6j9L+Xkfkz70rmOSAxxiVbPXpJZYqx1KZ5O4xjjjmG6dOn89VXX/Huu+/SpEmT7T0k6UfPALokSapKWzQbiYnRMcb2QBOS89+eJHmo9HwIYVvsllPwMGe3MvKbligHyQqAtUDv9FaC+/NNkLwguH4E35x/PilDnytjjFnlpGoxxk9K1Nua2V1BVK8yZ52Xkr6/3wJzgNYxxjNijL+PMY5In+1X6hXoLfkeY4zTY4z9SB7idSfZ8r8J8HAIoefWjFnfHx//8WNmHzmbNdOTIPrmrzYz5+g5zBs0j9wVW7J5xbbRpUsXWrduzb/+9S9Wriy9a+brr7/O/Pnz6devHw0aJM90hw8fztFHH82MGTM49thjueiii2jVqhU33XQThxxyCF999VWxNkIIbNq0icMPP5znnnuOvn37cuGFF9KiRQsA+vTpw4gRI2jQoAHnnnsuF1xwAd27d2fatGm8/vrrFd7DqlWr6Nq1K7///e9Zv349Z511FkOGDGGfffbhvvvuY+7cuYVlP/74Yzp27Midd95Jq1atuOyyy+jbty/PP/883bt359lnn63U5/b000/TrVs3nn/+eY466iguvfRS2rRpwx133EGnTp1YsGBBqc8ghMBvf/tbRowYQbt27bjooovo1q1bpfqTJOn7LL1j1wnAYyS7Xk0OITText0WzG3ermhOUqTOOSSrw4fFGLvGGH8TY7wqPSf4e1UMKsbYK8MYCo4E+1bzmrSCvY/Lmlc2KHkhxrgpPfdpC+xOsq39VOAM4PEt6LvgM69XwRxw6ha0qR+5zas3s/zZ4i8fx/zIkkeW7HDB4ZycHAYMGEDz5s3Jzs7mJz/5CX369OHxx8v/z6hnz56kUpnDVmPHjiWVSvHAAw8Uuz5r1iwGDBhAixYtyM7OZpdddqFdu3ZcfPHF5OXlAdCyZcvCl5YL+ijYJr2oDRs2MHLkSNq3b0+dOnWoW7cu3bp149FHHy01nilTppBKpRgxYgTTp0+nb9++NGzYkKysLBYtqswJHd9o1aoVnTp1ombNst5LkvRdcwt3SZK0Q4gxLgeeAp5Kv9nfiyRQPauKu3qbJODdk+QcvkIhhCySLdgB3ioytrz01u5Hk5xVHkkH0GOM80MIn5AE0GuRPKR5uUSfbwDHhBD2iTHOZdt6A+gI9AE+2Ir6e5K8ZPlijHF90YwQQvN0fpkq+z2mHxy+AbwRQphPsuL9eDJsra8dz7J/LOPrJV/T7Lxvnmeum7uOxXcvZq8b9yJkZd4Wr83/tWHO0XOYfdRs9n9yfz6++mPWvbOOdi+0o3qj6t/V8Ms1cOBA/vjHP/LII48wZMiQYnljx44lhMDAgQMBePnllxk+fDiHHnoozz33HHXqfLOA6IEHHmDQoEFcffXV/PWvfy3WzhdffMH+++/P1KlTqVWrVuH1d955h+nTp/PLX/6S8ePHlxpbphXtJQ0ZMoR33nmHCy64gL/97W/F8tatW1f4AAng17/+NUuWLOGGG27gsssuK7x+3nnncdhhhzF48GAWLlzITjvtRFnWrVvHmWeeSYyRyZMn06XLNzuU3nDDDfz+97/n17/+NRMmTChWL8bIrFmzmDVrFrvvvnuF9yVJ0g9Jen5xKrCR5NikV0IIvYueG17F/a0LIbwL7BdCaBBjXFVhJdiLZN7zZIa8nlU5vjK8STK3OjSEkL2V27gXvBH505IZIYS9SXYPK/Os8fT38QjJdvsfAD1KfH75FF9RX9QbQAegB8kOXdK39vmdn/PRFR/R9r627DZwN2J+5INff8Di/1tMdots6netX3Ej34G7776bIUOGUK1aNfr160erVq1YunQpOTk53H777Zx00kll1q1om/WSebNnz6Zr165kZWXRr18/WrZsyZo1a5g/fz533HEH1113HbVr12bo0KE89dRTTJkyhUGDBhW+wFy0vdWrV9OrVy9mz55Nx44dOeuss8jPz2fChAmceuqpvPfee4VB+KKmTZvGtddeS48ePTj77LNZunQpNWqU3LhP0vdOjNFkMplMJpOp0onkIUFeiWt7pK/fW069l4vWI9kGvFuGctVJgtx5QNtKjKfMvkkeRuUBZxS5thOwHPga6FKi/GXptv6doa1L0nlfkKyGSBXJu5dkhcF6YEaGuoen674KNM2QXzvDWD4GPtqK72ef9L0tA9pkyG9W3mdHsho8H3i9xD3WIXnwU+z735LvkWSbwuwMZQs+9+u39/+/f8yJ5AFfnDlzZqzIh5d+GF/m5fjpmE9jjDGufW9tfLXJq/HN/d+MX6/8uty6uWty45vt3owv83J8mZfj6jdWV9jfJ3/7JP73iv/G/Pz8wmsrX1kZ3/nlO3Hz+s0V1t8Sn3zySczKyoqdO3cudn3Tpk1x5513jrvttlvMy8uLMcZ4/PHHx1QqFefOnZuxrfbt28cmTZoUu9aiRYuYSqXinDlzSpV/5513YgghnnbaaRWOc8GCBTGEEAcPHlx4benSpTErKys2a9Ysbtiwodz6n376aQwhxJYtWxbeT1Gnn356TKVScdy4cYXX7r///phKpeLYsWMLrz344IMxhBDPOOOMUm3k5ubGli1bxlQqFRctWlR4fdCgQTGVSsXRo0dXeJ8lzZw5M5I8zO8Qd4D/bkwmk8lkqmzKNI9KXx+TzvsvsHuGOpPKaK/M+ULJuVf62uB0e/8A6meo0wBoX+Tvy9Plf1OiXPv0fCiv5ByM5AXlvJL3kc47LN3eVVvwmT2YrjM6Q95OJKu7y+ybZPHYKuBLYJci17OBf6Xb/qjI9V2A/TP0VZfkXPmvgTpFri8p5ztoQ7J71zygVYb86sAhFX1vpu9P2pL51FfvfBXnXzY/5ud9M7/5+suv47xz58XcVbll1svPy4/zzp4XXw4vx8/v/bzwfy++f3GFfX5X3nvvvVi9evXYqFGjOG/evFL5n36azCEzzWdijLFnz54xlUplbDvTfOSSSy6JqVQqPvPMM6XKr1q1qtjfw4YNi6lUKk6ZMiVj+wMHDoypVCrefPPNxa5v2rQp9u3bN2ZlZcVZs2YVXp88eXIMIcRUKhXvvvvujG1urYJ5Y6a5WkWcM5lMVZNcgS5JkraXWsCr6dXHM0nO1s4GjgTaAs/GGOdVQT/FXk+OyeqLM0m2HZwSQngcWESyavsokgcj52Vop2DL9l2Bf8YY80vkDaLIyvQSfU4KIVwOjAQ+DCH8i+SBVx2SIPZhJNsCHrOV91i0r7khhCHA7cCsEMIzwIckD4M6kTxA6l1O/SUhhEeBAen6L5CszDgS2ECykrxdkSpb8j3+Djg8hDA1ff9rgf1IVvZ/Cdz1be9f3429btyLuDny4ZAPWT9vPUsfW0qNXWvQblI7qjeo+pXkMTey6PpkC7yW17Vk9aurmXP0HOp1rrd1Bx2Uo3nz5hx++OG89NJLzJs3j7Zt2wLJNuWrVq3i0ksvLdxS8I033qB69eo89thjGdv6+uuvWbZsGStXrmTnnXcuvJ6dnc3PfvazUuX33Xdf2rdvzyOPPMKiRYvo168f3bt356CDDqJ69Yo/15ycHPLz8+nRowfZ2dnlli04T/3QQw/NuEVi7969eeihh3jrrbc4/fTTy20nhEDPnj1L5VWrVo0ePXowbtw43n77bX760+KLvzp37lzhPUmS9EMXYxwSQtgADAWmpleiz98G/dwXQugADAH+G0KYQDIPagi0JFkpfW86H5Idov4fMCqE0ItkTtEKOBYYD5xS1WPM4Dck84ULQgi9gRdIgth7kszdfgG8UlblGOPmEMJfgatJ5jb/IAmqH0lytvrnJao0A94OIbxDcqTVJ0A9kntuAvwtxri2SPmXgAEhhKdIXh7eDLwSY5waY3w/Pe/8P+DdEMK/SXYIq06yNfyhwFJg3636ZPS9tm7OOj75yydsXrWZ1ne2ZvPqzcw+cjYbP95I8wubU61+5rBNSAVa39maGCPvn/k+AG3vT1ajl2flyyv5fMzntB3XlqzsZNOEjQs38v6579Pm/9qQ3bz8ucOWGDNmDHl5eVx11VW0adOmVH6zZt/mVIbSClaQF93Zq0D9+pVfkf/ll1/y0EMP0alTJy6++OJieTVq1OCGG25gwoQJPPzww7Rr165Y/oEHHsjZZ5+9FaOXtCMzgC5JkrZGppBVLON6WfXWkQRUe5GsTD4O+Ipk5cV5lNhevRLtltV3qesxxmdCCN2BP5A8eKlPsrJ8DHBNjPGLDHVmhxCWA40oHSQvOPM80/nnBfVvDCFMAy4EDgH6kaxa/wy4g2RrwArHXhkxxnvSD30uIwnOH0ey6n4OcE+GPkr2cybJ9zCA5AHaMuBpkgdPT7L13+NtJIHyLiTnn1cDPgVuBf4aS58Brx1UCIG9b96b9fPW89noZKfRTu90osau5W9TV3Dm+caPN9JuYjs+vvpjZh81m3YvtKNel3pl1vvp0CTw+t9L/suqqatYO2st9TrX42f//BlZtcvaNXPrDRw4kIkTJzJ27FhGjhwJfLN9+xlnnFFYbsWKFeTl5WXcxq9ACIG1a9cWC6A3bpz5mNNUKsWkSZMYMWIETzzxBJdffjkxRurVq8egQYMKtx8sy6pVyW6ilXkoVbAd/G67ZX7Y1rRp02LltkU7ZdWRJOkHLOPv+xjjpSGE9STzkykhhCNicvRTRXOsLcqLMf42hPA8ye/03iSrzr8kCaTfADxUpOziEMIhwPUkv92PIllNfR7JnGdAGf1XNKZKz3FijKtCCN2Ai9P9nUOyyvwTknnNexX1HWMckf5sf52u/wXJ3Gs4MLdEnQXAVSRb1PckeQn5S+B94HcxxpJnv19Esoq9N/BzkqOwhpO8HE2M8aEQwizgUpL50pEk86fPSc5Tz/QW5o51kLW2iSanNiHmRuYNnsfmVZvZ8NEGNi7YSLuX2rHTfmUfn7S14ubIin+u4N0T3mW/8fuRuySXWb3Sp6zll193S02fPh2Avn37Vm3DZTjllFO45ZZbOO644zjppJPo3bs33bt3Z889yz19rpScnJzC466GDx9eKv/rr78GYN680us8fDFY+mEygC5JkrZIjLHUUsUY40LKPvutoEyvEn9vBm5Kp28znjL7jjGOBcaWkTcTOGEL+8oY9YoxLiZ5WFJR/deA1yrZV8stGVuG+tOBsg8Wo+zPLibnC/4pnUra6u8xxjgRmFhROX0/rJ+3nrWzvlmAs+yJZTQ7v/zA7ftnvV945nm9LvWo27kuc46ew5y+c+gyv0u556D/dOhPWf3qapY/uRxgmwXPAfr378+QIUN48MEHue6661i+fDkvvPACBx54YLGV4/Xr1yfGyPLly7eo/fLO9Ktfvz5/+ctf+Mtf/sJHH33ElClTuPPOOxk9ejSrV6/mvvvKfreoQYMGAHz2WcXHpxasxvjii1LvCwGwePHiYuW2RTvlfQ6SJP3QZJpHlcgv9fs7xljmj53y5gsl514l8v5Fsn15hdI7SR1fRnamecRgkq3iM7U1JVOdSoxhA8lOXiMrKFde32XNV1qWKLcauCadKjO2ZUDZW/UkZd4leUG5Mu2V+b3ph2e3gbuRuzKX/w79LwAdZ3ak7oF1y61TcOb5F/d+QZt727DmtTXMGzyvsL2yNDyyIfs/vT//Oe4/vNXpLXK/zCVVM8WBLx9I9u5Vt/octuyl3qpw0EEH8eqrr3LttdfyxBNPMG7cOGKMtG3blmHDhnHyySdXqp0VK1YASSA9JycnY5kQAuvWrSt13ReDpR+mCh/0SpIkSVJR6+auY1avWdTYtQbdlnSj2UXN+HDIh3x2e/mB25bXtKTdi9+sNq9WtxoHPH8Abe9vW27wHGDV1FV8OeHLwr8X/nkhMW6bBTq1atXixBNP5PPPP2fixIk8+OCDbN68mYEDBxYrd/DBB7Ny5Urmzp27Tcax5557MnjwYCZPnkydOnX4xz/+UW75zp07k0qlmDp1Khs3biy3bPv27QF49dVXyc8vvexk0qRJhBDo0KFDhe3EGJk8eXKpvLy8PKZOnQpQYTuSJEnSj0nuylyWPLik8O/Pb/+cmF/+/OaTmz5h8f8tpu19bWk6uCmt72xN07OaMm/wPFa/Xv7OUQ2Pasjef9ubdf9Zx9eff80B/z6A7D2qNngOW/ZSbyYFx0tlmqMUBOdL6tKlC8888wwrV65k2rRpXHXVVSxZsoRTTz014zwlk4IXfocOHUpeXl6ZaeLE0msCfDFY+mEygC5JkiRpiywYtqDwzPMajWvw/9m77zip6rP//68zW9jC0gVHSgABsYOiELCA3fwUo8ZEjSIao181yS0msccgKmpu8rV8E0tMNJaIRk3Q3InR2wIqNsQWEVHa0oaysLss22fm/P74nLNTdmZ3dnfK7sz7+XicB0w7ZbbN57o+1/UZc88Yhv7XUNbdtI7mnc1xX1cyrsSsWx4mvyyfQWcMavN4VW9Xtax5fnTt0ez7f/dlw10bWHfjupQl0WfPno1t2zz++OM8+eSTFBQUcP7550c8Z86cOdi2zY9//OOWSutwdXV1LS0ME7F+/XrWrVvX6v5du3bR2NgYc12/cIMGDeLcc89l8+bNXHvtta0er62tZffu3YCpCDnxxBNZv34999xzT8TzPvjgAxYuXMiAAQM488wz2zzmd7/7XQYMGMDChQtbXes999zDunXrOPHEExk2bFib+xERERERyRXNlc0ta54f/snhjP/zeHx/8vH15V+3mUTf5/J9OOjFg1qqzd010ff/y/70mRJ/SSwwa55vmL+h5fbqOasJNASSc0FhpkyZAsArr7zSqde7S19t3Nh6hbd4leGugoICpkyZwty5c7nvvvsIBoMsWrSo5fG8PNMEw23VHi58MrKICCiBLiIiIiIiHTT+0fEc+uahLWueu2uiT/pkUruV5J2x9fGtEWueD58znH3/775sW7gNf6U/6ccDOProoxk1ahTPPfccn376KaeeeiqDBkUm+o877jjuvvtu3n//fcaOHcv3v/99rrvuOq666ipOO+00hgwZEnP9vHg+++wzxowZw5QpU7j44ou56aabuOyyy5g4cSJ+v5/rrruu3X387ne/49BDD+X3v/89Bx54IHPmzOG6667jnHPOYZ999uHTTz9tee5DDz3E3nvvzbXXXsvJJ5/MTTfdxIUXXsj06dPJy8vjscceo7Q0cg3G6AkLpaWlPProo3g8Ho499lguvPBCbrrpJk455RSuvfZa9tlnHx566KGE3wMRERERkWzn+6OPhnVmzfOyCWXsfdHejH9sPFv/vJXdH+6O+7r8vvkMOj1yTGJ5LIacN6TNKuiG8oaWNc+nrJ/CIa8cQtUbVaw4e0XSk+hXXHEFeXl5zJs3j1WrVrV6vL3K9MmTJ2PbNo888kjE/a+//jrPPPNMq+e/9957MbtvuUtMFRWFquwHDhyIbdsxk/N77bUXP/zhD/noo4+4/fbbY1bAr127lvXr17d5/iKSPbQGuoiIiIiIdEheaR55pZFLWFqWlZIWgADjHhqH3WRHrHk+fM5wvJd6yS9L3ZBm1qxZzJs3D4/Hw+zZs2M+55e//CXTpk3j/vvv55133uGll16ib9++DB06lP/zf/4P5513XqvXxAtuTZo0iRtuuIElS5bwyiuvUFlZyV577cURRxzBz372M0466aRW+4neV79+/Xj33Xe59957efbZZ3nkkUfIy8tj+PDhXHrppRxwwAEtzx01alRLgOhf//oXS5YsoU+fPnznO9/hxhtv5PDDD0/o3GfOnMnSpUuZP38+r776KtXV1ey9995ceeWV3HzzzTHXBFSbQxERERHJVcN/MZzBPxgcsf743hftTb/p/VIyptr5slnfe8KbEyj6VhFF3yrioBcP4svzvqRuRR1lh7e99npH7L///jzwwANcccUVTJgwgZkzZzJ27FgqKipYtmwZ/fr14/XXX4/7+osvvpgFCxZw55138umnn3LAAQfw9ddf8+9//5uzzjqL559/PuL5v/nNb3jjjTdaJkD37t2bFStW8PLLLzNgwAAuu+yylufOmDEDj8fD9ddfz+eff95S7X7TTTcBZjLy6tWr+fWvf82TTz7JUUcdxZAhQ9iyZQsrV67ko48+YuHChYwcOTJp75dr586d/PznP28ZJ1VUVABwySWXtNx3ww03MG7cuKQfW0Ris1LV8lBERERERHoOy7IOA5YvX75c61VLt/Xxxx+7if3Dbdv+ONPnIyIiIiIC3X885a/xt5p8HOu+ZPnggw9YsGABb7/9NlVVVQwaNIhDDjmESy+9lLPOOovy8nJGjx7N7Nmz+dOf/hTx2q+++opf/vKXvPXWW9i2zaRJk5g3bx5r1qzhkksu4bHHHmPWrFkAvPbaay3LSW3evBm/38+wYcM45ZRTuOaaaxg+fHjEvp9++mkWLFjAqlWraGhowLIs/P5QVzO/388f/vAHnn76aVasWEFDQwNDhgxh7NixzJw5kwsuuKAl8b5kyRKOO+445s6dy69+9asuvV/l5eWMGjWqzYnGb775Jsccc0y7+9KYSSQ5lEAXEREREZFuH/ARAQWDRERERKR70nhKuguNmUSSQ2ugi4iIiIiIiIiIiIiIiIiIoAS6iIiIiIiIiIiIiIiIiIgIAKlZ4EJEREREREREREREREQkx9x7771UV1e3+7zp06dz7LHHpuGMRKSjlEAXERERERERERERERERSYL77ruPDRs2tPs8y7KUQBfpppRAFxEREREREREREREREUmCdevWZfoURKSLtMkxRBMAACAASURBVAa6iIiIiIiIiIiIiIiIiIgISqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIiIiIiIiIiIiIgASqCLiIiIiIh0ypIlS/B4PMybNy/lx5o9ezYej4cNGzZ0aT9//vOf8Xg8PPHEE0k6MxERERERERGR7KIEuoiIiIiISCdZloVlWT3mOOk6XxERERERERGRnkoJdBERERERkU6YPHkyK1eu5Kqrrkr5se666y5WrlzJ0KFDu7Sfs846i5UrV3LmmWcm6cxEREREREQ6pry8HI/HwyWXXJLpUxERiUkJdBERERERkU4oKipi3LhxDBgwIOXHGjJkCOPGjSMvL69L+ykrK2PcuHGUlZUl6cxERERERES6v7lz5+LxeHjrrbcyfSqt/Pvf/+anP/0pEydOZODAgRQXF7PffvsxZ84ctm/fnunTE8lJSqCLiIiIiEhaNTY2ctFFF/Hxxx9n+lQifPDBB3g8Hs4+++y4zxk3bhzFxcVUVVXFXQN9+vTp5OXl0dTUxC233MK4cePo1atXRHXF7t27ufrqqxk+fDjFxcXsv//+3HPPPaxbty5mJUasNdDDqzbKy8s599xz2WuvvSguLmbSpEn84x//aHX+jz/+eNw10Ddv3szPfvYzxo0bR0lJCQMHDmTy5MncfvvtEc9bvHgxl112GQceeCB9+/alpKSEgw46iLlz59LQ0ND2mywiIiIiIpIB3XU5q8bGRr7zne/wyCOPMHjwYC655BKuvPJKSkpKuO+++zj00ENZvXp1pk9TJOfkZ/oERERERESk5woGg3g8HZuX++ijj/LEE0+wdm05b7+9ODUn1gmTJ09m3Lhx/Otf/6KyspL+/ftHPP7ee++xevVqzjnnHPr16xd3P25Q5uyzz2b58uWceuqpnHnmmQwZMgQwAZIZM2bwySefcNhhh3HBBRdQXV3N/Pnzefvtt2MGddoK9qxfv54jjzySfffdl1mzZrFr1y6effZZzjzzTP73f/+XGTNmxDy/cB999BEnn3wyVVVVHHvssZx99tnU1tby5Zdfcuutt3LzzTe3PPfuu+9m1apVTJ06ldNOO436+nqWLl3KvHnzWLx4MW+88UaHvydERERERERSybbtTJ9CTHl5edx+++1ceeWVrcaZV1xxBQ8//DDXXHMNL730UobOUCQ3KaohIiIiIiKdsmPHDkaOHM1f//rXhF/T2NjIvHnzgXG8884SFi9e3KFj1tTUdOwkO+iiiy6iqamJhQsXtnrs8ccfx7IsLrroonb3Y9s2GzduZMWKFfzpT3/i7rvv5pprrgHgN7/5DZ988gnnn38+H330EXfeeScPPPAAn376Ke+8806Hz3nJkiX89Kc/5d133+W3v/0tjz32GIsWLSIYDLJgwYJ2X9/c3Mw555xDVVUVCxcu5I033uDOO+/k/vvv57XXXmP9+vURz3/wwQdZu3YtTz31FHfffTf3338/y5cv58Ybb+Ttt9/m+eef7/A1iIiIiIhI4m688aZu/bl72bJl/OAHP2DYsGEUFRWxzz77cPLJJ/Pcc8+1+brp06fHnYwbr5vWp59+yg9+8ANGjhxJUVERgwYN4tBDD+Xqq68mEAgAMGrUqJbOYe4xPB5PqyWy6uvrufPOO5k4cSK9e/emrKyMqVOn8swzz7Q6n/COZB988AGnnHIKAwYMIC8vL6JzWHvy8/O58cYbY07SvuWWWwA6PG4Wka5TAl1ERERERDplwYIFbNxYzrXX3ojf70/oNY8++ijbtm0BXiQvbyK/+tXchI9XXl7O3nt7+f3vf9+5E07AhRdeiGVZPP744xH3NzU18de//pXBgwdzyimntLsfy7K47bbbWlWxgwn85OXlMX/+/Ij7hw4dytVXX93hyoiRI0dy0003Rdx30kknMWLECJYtW9bu6//xj39QXl7OGWecwfe///1Wjw8dOrTV8WKZM2cOtm3z6quvJn7yIiIiIiI5LBgMdvg1H374IXfeOZ/LLruS2traFJxV1zzyyCNMnTqVl156iWnTpvGLX/yC0047je3bt/Pggw+2+dr22qxHP/bZZ5/x7W9/m3/+859MnTqVn//855x33nnss88+PPTQQzQ2NgJmrDJ9+nTALI81d+7cls1VXV3NtGnTuPnmmykoKOBHP/oRs2fPpqKigvPPP78lmR1t6dKlHHPMMQQCAS699FIuvPBCCgsLE3in2ufup6CgICn7E5HEqYW7iIiIiIh02I4dO7j//t8B36W8fBFPPfUUs2fPbvM1oerz84DxBAJzeeedM1i8eHFLMKMtd9wxn7q6WubOvZ1LLrmE4uLiJFxJpGHDhnHcccfx+uuv89VXXzF+/HgAXnzxRaqqqvj5z3+ecHvyI488stV9NTU1rF27lhEjRjBixIhWjx911FEdPucJEybEDDINHz6c999/v93Xv//++1iWldDEAIC6ujruvfdeFi1axNdff01NTU1L0t+yLDZv3tyxCxARERERyUHBYJBp047huOOO5Y477kj4db/+9a3k5Y2kqmoTDz74IL/4xS8Sfm1tbS1FRUWtKq+TZeXKlVx11VX07duXpUuXst9++0U8nuyxwhNPPEFTUxOLFi3i9NNPj3isurqakpISAH72s59RWVnJkiVLmD17Nsccc0yrff3Xf/0Xn332Gb/97W+5+uqrW+5vamrijDPOYP78+Zx99tkceuihEa977bXXePjhh7n00kuTem0Af/zjHwE49dRTk75vEWmbKtBFRERERKTDFixYQHNzHvBHLOtM5s69vd0qdLf63Lbd9bRPT7gKvby8nEcffRS4gp07d/Dwww939RLiuuiii7BtO6IK3W3fPmvWrIT34655Hm737t1xH2vr/rb07ds35v35+fkJVbRUVVUBrSvNY/H7/cyYMYObb76ZxsZGzj33XG688caW6g3btluqPEREREREJL4XXniB999fym9+s4BNmzYl9JoPP/yQf//7XwQC87Hti5k//zcJV6E3NTVx8MET+dGPkp/odT3wwAMEAgFuueWWVslzSGzM0RHuROJYk6vjjZNi2bVrF3/5y1844ogjIpLnYKrA7777boLBIE8//XSr106YMCElyfNly5Yxb948ysrKuO2225K+fxFpmyrQRURERESkQ9zq80BgDjAQ276F8vKJbVah+/1+5s2bj20PBO5ruT8Q6MU77yzhrbfeilkF4LrjjvlYVn/gv7HtBu64424uv/zylFShn3XWWVx55ZU89dRTzJ8/n4qKCl599VUmTJjAwQcf3KV99+nTB4Bt27bFfDze/ankrrWXSDXIiy++yLJly7j44ov505/+FPHY1q1bI1ogioiIiIhIbMFgkF/96lY8nmOw7S+46667+N3vftfu637961vJzx+P3/994NtUVT2WcBX6448/zrp137B+/Rquv/66lm5byfTBBx8AJNzdqqvOPfdc7rvvPs444wzOOeccjj/+eKZNm8bo0aM7tJ9ly5a1rJd+6623tnq8qakJgK+++qrVY7E6j3XVqlWrOP300/H7/SxcuJBRo0Yl/Rgi0jYl0EVEREREpENC1edznHsmtFShX3DBBeTntx5m2LbNwQcfxF57bQc+jHgsP/+INteIc6vPA4E7gVLgJnbufIKHH364VXVAMhQXF/O9732PP//5z7z22mt88cUX+P1+Lrrooi7vu6ysjNGjR1NeXs6GDRtatXF/++23u3yMjpoyZQq2bfPKK69w2WWXtfnc1atXY1kWZ511VqvHFi9enKIzFBERERHJLi+88AKrVq0AlgKLefjhW7n++usZNmxY3Ne41efwNJAHjGypQr/iiisoLS2N+9qmpiZuvfUOLOu75OV9xG233c5f/vJUkq+qY92tkmHSpEm888473HHHHTz//PM8+eST2LbN+PHjmTt3Lt///vcT2s/OnTsBk0hftmxZzOdYlhWz2n/vvffu/AXE8NVXXzFjxgyqqqp49tlnmTlzZlL3LyKJUQt3ERERERFJWGNjI/ff/zuCwQAFBUdRULA/BQX74/G8R3n5Gl566aWYrysoKODVV1/m88+Xt9o+/vhDpkyZEveYoerzK5x79sW2Z3HHHXdTX1+f/IsEZs+e3dLG/cknn6SgoIDzzz8/KfueNWsWgUCAG264IeL+jRs3ct9998VczzyVTj/9dEaOHMmiRYt47rnnWj0eXpk+cuRIbNvmzTffjHjO2rVruf7669N+7iIiIiIiPU2o+vwkYCrwE2y7N3fddVebr5s373bAAt7EjI2uALZSWdn+ElePP/44mzdvwLZvw++/gYULF8aspu6qjnS3isXjMSmrWEtRucn5aJMnT+all16isrKSpUuXcsstt7Bt2zbOP//8hCf5uu3e58yZQyAQiLu99tprrV6bzDHQihUrmD59OlVVVTz//PN897vfTdq+RaRjVIEuIiIiIiIJKygoYO7cW9i+fXurx/Ly8jj88MOTeryGhgYee+wx/H4/Hs+glvtt209FhZ9FixZx3nnnJfWYAEcffTSjRo3iueeeo7m5mZkzZzJo0KD2X5iAa6+9lkWLFvHMM8/w1VdfcdJJJ1FVVcVzzz3Hsccey9///veWwFEq2LYdcbugoIDnnnuOk08+mR/84Ac89NBDHHnkkdTX1/Pll1+yePHilpaFp59+OmPGjOGee+7hP//5DxMnTqS8vJx//vOfnHbaaTzzzDMpO28RERERkWwQqj7/g3NPHwKBn7dbhT5q1CgOPngisDzqkcPaHKuEqs/PwbYPAsaSlzc/JVXoU6ZMYfny5bzyyiuMGzeuw6/v378/YCYXf+tb34p4LF5luKugoIApU6YwZcoUxowZw6xZs1i0aBHTp08HzHgVaGnVHu7II4/E4/FkpCOY6/PPP+eEE05gz549/P3vf09bG3wRiU0JdBERERERSZjH4+G6665L2/F69erFU089GTdhf9JJJ6Xs2LNmzWLevHl4PJ64a7tblhWz4qCtKoSioiIWL17MLbfcwvPPP8+9997LqFGjuPnmm5k2bRp///vfWyog2ttnvOO395pohx9+OJ9++il33XUXL7/8MkuXLqWsrIwxY8Zw2223tTyvpKSEN998k+uvv57FixfzzjvvMHr0aH79619z9dVX8+yzz6oKXURERESkDQsW3ANYFBT8qOU+227A72/igQceYP78+TFf9//+332dOp5bfQ7/49zTC7//RhYu/Cm/+tXNSV0L/YorruChhx5i3rx5nHTSSey3334Rj2/evLnN9u6TJ0/mb3/7G4888gi33357y/2vv/56zMm67733HhMnTqSoqCji/q1btwJE3D9w4EBs22bjxo2t9rPXXnvxwx/+kKeeeorbb7+dG2+8sdWk5rVr1+LxeBg5cmT8N6CTPv30U0444QQaGhp46aWXOOGEE5J+DBHpGCu6+kBERERERHKPZVmHAcuXL1/OYYcdlunTyVmPPPIIl19+OQ8//DA//vGPM3063c7HH3/sdjk43LbtjzN9PiIiIiIi0LHx1FNPPcUnn3wS87FzzjmnzeWtOmPChMP47LNP8HhCyWTbDmLbTVxzzTX89re/Terx/vjHP3LFFVeQn5/PzJkzGTt2LBUVFSxbtox+/frx+uuvU15ezqhRo5g9ezaPPvpoy2srKio48MADqaio4NRTT+WAAw7g66+/5t///jczZ87k+eef589//jOzZs0C4Mwzz+SNN95o6SDWu3dvVqxYwcsvv0zfvn358MMPGT16NGDWFj/ooIMYPHgw559/fku1+0033QRATU0Np5xyCu+//z5jxozhqKOOYsiQIWzZsoWVK1fy0UcfsXDhwpZ11ZcsWcKMGTOYO3cut9xyS6ffr6qqKvbdd1+qqqo4/vjjmTZtWsznzZkzhz59+rS7P42ZRJJDFegiIiIiIiJp5vP58Hq9Efdt2LCB2267jcLCQmbOnJmhMxMRERERkVS64IILuOCCC9J2vHvu+S1ffPFFzMdOPvnkpB/v0ksv5eCDD2bBggUsWbKEF198kUGDBnHIIYdw6aWXtjwvVjetQYMGsWTJEn75y1/y1ltv8dZbbzFp0iRee+011qxZwwsvvBDx/KuuuooBAwbwwQcfsHTpUvx+P8OGDeMnP/kJ11xzDcOHD2957vjx43niiSdYsGABDz74IA0NDViW1ZJALysrY8mSJfzhD3/g6aef5m9/+xsNDQ0MGTKEsWPHcu+993LiiSdGHL+9jmCJqK6ublnf/fXXX+f111+P+byLL744oQS6iCSHKtBFREREREQV6Gk2depUmpubOfzww+nXrx/r16/nf/7nf6ivr+e///u/ueaaazJ9it2SqilEREREpDvSeEq6C42ZRJJDFegiIiIiIiJpdtFFF/GXv/yFRYsWUVlZSe/evZk6dSo//elPOf300zN9eiIiIiIiIiIiOUsJdBERERERkTS7/PLLufzyyzN9GiIiIiIiIiIiEkUJdBEREREREREREREREZEkuPfee6murm73edOnT+fYY49NwxmJSEcpgS4iIiIiIiIiIiIiIiKSBPfddx8bNmxo93mWZSmBLtJNKYEuIiIiIiIiIiIiIiIikgTr1q3L9CmISBd5Mn0CIiIiIiIiIiIiIiIiIiIi3YES6CIiIiIiIiIiIiIiIiIiIiiBLiIiIiIiIiIiIiIiIiIiAmgNdBERERERCbNy5cpMn4JIXPr+FBEREREREZFUUwJdREREREQAKjweT8MFF1xQlOkTEWmLx+NpCAaDFZk+DxERERERERHJTkqgi4iIiIgItm1vsCxrP2BQps9FpC3BYLDCtu0NmT4PEREREREREclOSqCLiIiIiAhgkuiAEpMiIiIiIiKdoCWHJNP0PSiSHJZt25k+BxERERERERERERERkR7JsqwRHo9nVTAY1JJYknHOslf7qXOXSOcpgS4iIiIiIiIiIiIiItIFlmWNQEtiSfegZa9EukgJdBEREREREREREREREREREcCT6RMQERERERERERERERERERHpDpRAFxERERERERERERERERERQQl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERQAl0ERERERERERERERERERERAPIzfQIiIiIikt1qamrygBKg2Pk3+v/u1jcQCAxraGhosG17BbDJ2bZ6vd5ARk5eREREREQkxWpqaiygF7HHS+H/72Pbdp/Gxsahfr//S6AcZ9zk9Xr3ZOTkRUREspBl23amz0FEREREeoiampoC2k6EFwOlQD+gj7P1BgowkzcLwv7vfhC1nH+b/X5//4aGhgKg0rlvD7AL+BpYRyipvtvr9eqDrIiIiIiIdCs1NTUeoIj2JxGXYcZNfTFjpl6ExkvuZhE5bgrYth2or6//VjAYrAQagCZgN7ARWEXkRGR/qq9XREQkGymBLiIiIpKDnAqHAtpOhLtBnT6YoE4f576WRHh9ff20/Pz8LQUFBVucXbsBnmZn84f9vznq/laam5u/1djYOAhYjlluqDehRLwHGAtsAT4AvsIEiTYBW7xeb3MS3hoRERERERGgpZtWMQl003K2MmcrxBkzNTc3j/L7/UOLi4uXh+3aAgLEHiuFj6OC0edk27anvr5+cjAYXAtswyTe3TFTMTAQGAS8RuuJyNWaiCwiItI+tXAXERER6eGi2v3FS4SXYJLRboVDH2JXOHiIrHAI0joR3gDUAM2BQMBrWdY3BQUFK5J4SW5FehBTSbE77LHjnXMYDRzk3FcHVPl8vtXAGkLBoV0KDomIiIiICEBNTU0+iXfTKiNUGR49ZsqL2rVF7InD9YA/EAiMCAQCg4FkjpnCNQI7nA3gSOAA4H1gOmYM5ceMq3w+ny96InJTis5LRESkx1ICXURERKQbcZLh0RUOsVr/9aET7f5oHdipI7LKoSesNV4PrHX+b2GCXH2AqcAxhBLv25zg0AZMcGiz1+ttSP/pioiIiIhIMjlLS7XXTcvtZuVWh5cQOV7KJ87SUlFbE63HTR2ZqNuqijwJ2jq++9hmZwNTEV8GjAQOdO5zJyKvIXIi8k5NRBYRkVynBLqIiIhIijhr37WXCC+ljXZ/YRu0ToZHB3b20E67vxRJdnClvf3ZUf/f42xuG/l8TKBsKLAf5v2qB3b7fL61wDeEgkM7vF5vut4nEREREREJ40wgLiSxblp9w7YiIhPhbmV4+JgpfGkpd2skNG6Ku7RUCiR7zGS1/5QITcBOZ3NfX4IZN00BjiY0EXl7jInI9ck4aRERkZ5CCXQRERGRBLTT7s/9f6lt2wMCgcAoj8djezyeWInwDrX7C7vdXSsAOhq4Scfx/MAuZ3O5waHDMAEiG9OGvsIJDpXjJNW9Xm9tUs9YRERERCQHOMnwIhLophUIBEYA/fLy8gKEkuFuIrytpaXCE9/1mISve19P6KaVTPHGiFYbj4W/ttbZfM59+ZgJ3V5gLOZr4E5EXkfkROTtmogsIiLZTAl0ERERyTlttPuLrnDocLs/27Y9DQ0NZxYWFr5WWFi4hq63+8s1qUrI1znbVud2HiY4NAg4lVBV/0Cfz7cM+IhQcGir1+vNtWCciIiIiOSwsG5a7U0idsdLfZytkMhEeMylpZqamiYFg8HS0tLSlzBjpFoiE+RKzsaXqjGTH6h0Npe7fNgEYDLm61gM1Pp8vv8F1hOaiLwnReclIiKSdkqgi4iISI/VRru/6IqH8HZ/ZUQmw5Pd7q/U+beCULJWOqatCQaJVFMkIgBUOZtrMPB95xhjnfv2ALt8Pt/XwDpCSfXdWhdQRERERHqCmpqaPNoeL4UvLeVOIu5N625asZaWCu+a5f6/Ieq+Vslw27b3w4zFtkQ/loUy0bUrWWOVemfb5tz2AD/AfH9Yzu0mTJX6RmAVkROR09UiX0REJKmUQBcREZFuIU67v1iBHTeg41Y4qN1f5mV6Pb9kaXT+3QKswXwfuZ0IjnNu+zHfN1uc1u8bMcEhn9frbUr7GYuIiIhITnGWlmprvFRCKBnujptKaZ0ID++m5XIT4OFJ8XrUTSsZsuV9C2K+DxqAL537emG+z8YDEwm1h6/0+XzfAGsJJdWrNRFZRER6AiXQRUREJOmcdn9FtL3uXXS7vzLMwLvddn+0rnBQu7/MyVSyOxXca3G/34KYZPnusOcUYr5XRwEHOffVAVU+n281JvHuBod2KTgkIiIiIrE4E4jdpaXaWjO8FOhHaNxUTOtkeLxuWuGJ8CYix02qDJauCB/nNAI7nA3M96A7EflYzGRkdyKyL2oi8hZNRBYRke5ICXQRERFpUzvt/qIrHFq1+/P7/Xs1NzcfUFRU9IVlWeEVC+Ht/sIDO+22+5Osl44W7p3VBOx0NjDnU4L53v82cAyhxPs2Jzi0ARMc2uz1ehvSfsYiIiIiklJOMrwXbSfCizETMfvRuptWQUNDw+Eej6emsLBwc9iu3W5a4eMld8xUQ2jMpGS4hEv1mKm9SdQ25vuzBnC/nwsw3/MjgQOd++qA6hgTkXdqIrKIiGSaEugiIiI5pI12f9EVDp1t9xcd2KkPBoP9AoHAvsFgcFFeXt6elF6gZItMVbVHV6Anwm1PWAv4nPvyMcHRfYBxzn4bMOsCbsasC7geExza7vV6NUlEREREpJsIW1qqvW5a0UtL9aL1mCl8aSkIVYaHb3VAcyAQ2Me27a+BFam9QsmAVI1vMplk7uixm4k/EXkKcDSm21wNsNPn860BviY0Ebk+GSctIiKSKCXQRUREeqA22v1FVzyEt/src+5LpN1fdCI8vN2fmyhPdMCsitvsl8zATXvVEpmuQE+EH6h0NlcxJjh0A6Zd4X+APUCFz+dbBZTjVFx4vV5NNBERERFJAmdpqfa6ablLS7kJ8TLMsj3hifD2lpZytz1EjqU6MlEySPf/nCudl01jpmRMCGhrIvIxwC+BvxGaiLwO+IZQlbomIouISEopgS4iIpJhcdr9xWr957b7K8MEdlra/YVt0RUO0Dqo47b7C2+fLtJZ2bwGejLVO5sfszbgKsxSBwOBkzE/u02Y4NBG53E3OLTV6/Xq51RERERymrO0VCLdtNxEeMTSUoQS4QVEft4LX1oqetwUflvJbekU287Kb51UXFT4ROQg8AWhicgTgMmE2sPvcpbLWk+oSr0mBeckIiI5Sgl0ERGRJIrR7i/eGnhumz+30qG9dn/u2nex2v2FB3sCKb7ErsimRKtIVwWAamdz9cL8PhgPTCRUlVHp8/m+BtYRSqpXa11AERER6alqamoS6aZVQmQ3LTcZHp4ID19ayh1vRI+ZWtqkE5pA3F0/R2nMJD1BOr9P3YnI25zbHszvg/7ASWgisoiIpIgS6CIiInFEtfuLlwgPX/uu3XZ/DQ0NE4LBYL+SkpI3aB3UqY26nS3tyLprcCpVcu160y2V7QhTWYEefox4+2/EVKfvcG57CFVRTQeOxwR8dwM+p+JiIyY4tMXr9TalUYGNtgAAIABJREFU7rRFREREWnMmEBeSWDet8PXCS4hMhLcsLRUIBErr6+tPKioqeis/P38rkYnvRkyb9PD7soXGEdkr6Qnndqra07HsVabGTEE6NhF5NbCWUFK9ShORRUQkEUqgi4hITghr99dWItxd+84N7JTRuXZ/fuK0+7Nte5Rt2yXAypRdrEj6KQCRuI4Es4KY9oQ1wGbnvgLM76eRwIHOfXVAlc/nWwOsIRQc2qngkIiIiCQqbGmprnbTil5aKrqbljt+cpeWiuimZdv2AOBU27bLgfIUXrJIOmXT5/JUV6B3dAJA9ERkC9Oxog9mPfUZhCYib42aiLxZE5FFRCQWJdBFRKTHcdr9tZUIL8ZUbXa13Z8b2Elmuz+b3G3Ll6vXne3S/XXt6RXoXdUM7HQ2MOfsBrKnYKrUDwU+AlY5waENhIJD9Wk/YxEREUk7p5uWu7RUW2OnMkLjpt7EToaHf/6yMInu8PFS9JipS920LMuKHqPlkly8Zum8eOOWnl6B3lXuOumxJiJ/C9gfGOM8b6nP51sLrCY0EblCE5FFREQJdBERyRinwsFd+y5eItwN6iTU7i/qENGJ8O7S7i/XgiIaeErCbNtOR7Ank5J9fW57wlrAh/ldeTrwMeAFxmIqwOox6wKuA74hFBza7vV6s2W5CBERkazkdNMqpmPdtHoTubRUW920ojtqRY+ZMvVZQeMmkdgy/bOR6jFbKvYfPRH5AEzhRSFwJDCNUAew7T6fbxWmA8YmYJMmIouI5B4l0EVEJCnaafcXUeFg23Y/v98/Pi8vz+/xeGwSb/cXHtiJ2e6vB8jloEimB/nSTdm2bdm2XRwMBkuCweBgTCAjj/T/XGd6DfRk7R/M78fwlqfFmGD6BGAyoaqMXU6V+npCwaE9KTw/ERGRnFZTU5NP2+Ollm5agUBghG3bA/Pz8+tpPWbKi7F7tzI8fKsncizVE8YjPeEcU0VjpuyUlK+rbdsFzpipj7PUQUky9tvZ00nhvtMxqdqdQORzNjC5kjJgb0yFuoUpxNjt8/nWA18Tmoi8TRORRUSymxLoIiLSShvt/qIrHDrb7i/Y2Nh4bmFh4eLCwsKVtG6R3lOS4Z2Riy3cczkAlitavsa2bWPbdqFt2yXBYLDEtu1S27ZLwjcg+t9iWv9cfBvYglmbzt1qSU8wpSeLNwmg3tm2Obc9mN/h/YGTnNtNmODQRmAVoeDQVq/Xm6mOHSIiIt2Ws7RUWxXhxZhxklsV3te5P3y8lE/k0lJg/p43NzU1TQgGg0Py8/P/ivk7XUsoEd7VpaW6MxtaOhPlkmz9eooR8fW1bdvjjJfibaXRYydax/KPAKoxY6UNzr/bSd+k3Z4s1nvkByqdzeVORD4E837bmC4dO50q9fWEJiLXpPicRUQkjZRAFxHJcnHa/cUK7ISvF15GYu3+YlU4uO3+3MBOrBm5HgDbtqswg7tckw2DTckRtm3nhQd23P9jAjmlgMfv90+ura09PCywE6siqgmosyzL3Soty9ocdrvOsqy6QCAwwO/398NMrBkOHAhMdfZRifndNBQYDOwguYGhbKhAd7V3jCAm2FYddl8vTHBoPDCRUHv4Sp/P9w2wllBSvVrrAoqISLZwumkV0vZ4yZ1A7CbC+zj3hyfCo5eWcv/uR4+ZGkhsaalRwEBga7KutYfRuEl6BKejVq94E4gDgcC+QGltbe2PnDFTKeazd7QgzpiJ0NhpF5HjqFrLspoaGxvHYX6P9MOMmw7AxFrcuEwBpop6E+Z3TtIvOwX7dHWXMRO0PRH5RELv+W6fz7cJ+IrQmMmnicgiIj2XEugiIj2I0+6vvUR4KWYA5a4X3pv22/25ratitfsLvz9ZA5jwgFKuyeWEUy5+vbsVJ7BTFAwGWwV1oqvDneR4CSaYHC0I1DqBHcuyrCaPx7MpLNBT6/F4WoI8zv8T6ixh23Y+5nvlE+BD5+4+mKDQcMys//0wSd5GTGDCrVDfhEnUd3fdtRqkETMpYUfYvnpj3v9jgeMwfxN2AzU+n+9LQpXqW7xeb09470VEJMs5yfBY3bRaLS1FaM3wMuc14YnwtpaWih4z7Q67ncxuWrnYvQo0Zsol3fJ63Vbp4ROIw8dIsTprEftaGjATg/MBLMvaET5Oihov1VmW1WBZ7b8ltm0XNDU1DbNtew2hNb3zAS8wAjjUuf1D57EdhCrUNwK7OvfOtEjH1y0dLdw7o62JyOMw7z2Yich7fD6fD1hOKKlepYnIIiI9gxLoIiIZ0k67P/f/HW335w4AooM6TbRuk57JD+y5nkDPqeu2LCtXB4cp/TrHaJXeqtqBxFqlA9Q7ye9aJ3CzrY3ATq1lWU1uYGfPnj2/ysvL+6SoqGhZki4t1vntBlY426HAu5gAkJtUnwxMx/x8bSeyhWFVJ47dXRPcHdl/Mq7BXSe9Btjs3FeA+Zv0I8x7/Tnm70u1z+dbDawhFBzaqeCQiIh0hbO0VHQ3rVhjp75EJsMLiUyEF0TtOlY3LT8m4RF+X6bXt825sYMjV8eL+tyUAgm0SnfHSqVhyfBYMfNmIqvDd1uWtTWsKjx63FRvWVYQoL6+/vhAIHBQSUnJS0m6plg/G35CCXIPMAX4E6Ex03DgcOe5dUQulbWF+J0w4p5Gh088cen42U9mlXu8iciTgCuBFzB/c3YDW30+30pCE8C3eL3exiSdh4iIJJES6CIiSVZTU1OCaZMVq8Khpd1fc3PzKNu2excWFm6hY+3+Gkms3V+3FTajOtcCIq5cvW5pQzut0uOtgZdIq/Qqy7K2RAd1ogI7XQocpGCSRCLtx8udzTWIUGBoJCZYAeb3ZXhwyEdyK8M6Kl3rEabqGM2YKhc/UIGZ1FCC+fs2BTga8/7WADuc4NAGTHBos9frrU/ReYmISA9TU1MzGtMCN7qblpsI7xMMBgc0NzfvV1BQsMXj8fiJTIaHj5vCu2a5/2+Iui/TyfDOsHNwHfBwuXztuaDDn1ejWqW3OYk4bMxUFOfYdZZl1RIaO1USOY5qmUDs8XjqLctq7tLVZmaCxC5n+8y5XQQMIzRuOhbzOzWISaKHj5v2tLHfdEwKTkcFeqr+LrgTkd0q9S8ITUQeAezv3F+PmYi8BlhNaCJyhSYii4hknhLoIiLJNxgzw3QQbbT78/v9+wWDwUGFhYX/ITXt/rq7XK+myEU59fW2bbvQ7/cPjLUGXozATintt0p3AzkVUbfrPB5PrfNvvWVZ6Z5Uk4qva1v7jBdMqXC2T5zbxYSCQyMw7cfzMb9ntxCqUN+EqTYLP26qf067y3p+yTiGu056LWZyApj3uQzYGzOhzMIkMXb7fL51wDeEgkPbvV5vT0xoiIhI152C+fsc/TfLTYD7bdsubm5uPtbj8Tzn8Xg20T26aaWTxky5J6e+3rZtW4FAoG+MVunueCnWJOK4rdLDkt8V7hipK63Se7B4Y6YGTKJ2tXPbg4lhuQn1A4BvO49VEUqmb8B0nwrfp8ZMHeNORHZb7luEJiIfCUzDjP9rgO0+n28VZsK4OxG5Lo3nKiIiKIEuIpIKQUwybC2mLVY8dYQ+HOeinA0G5WAVSY8OgMVplR63KtxZF4/m5uYzm5tbFSnURyW/t7cT2GnM8sBOKtVjErXfOLc9mISuGxw6BBOkAFOVsZHIdexSJV0t3FOtraoQP1DpbK5iTHBoAqblvluVscvn830FrCcUHMrVv4siIrmoGVjVxuP9nX/raXtsla1ycswU1t0o1669p4+bPLZtFweDwZgTiGMsM9UHsOrr66+O2pWfyEnE4a3Sw7fa6Fbp0mFBYKuzuUtylRHZ9v1AzFiqCfN5fSMm8ZvKtuPpauGejmPE+7mONRE5D/P+DyE0EbkRMxF5PfA1kRORc6kIR0Qk7ZRAFxFJPrcir70P4zkZDAmTi4lkV65ed7eQQKv0WGvgtdkq3fm3yrKsLUBzIBA4Oi8vb2l+fv7XYQnxBgV2EtZe8LAzwUW3LeEW4APnvj6EKtTdpDrARYQq1Ddi1v9u6sQxY+npLdzDj9ORY9Q72zbntgcTHOoPnEQoKLfb5/NtxCRT3ODQVq/X2+OWKxERkXYlOmaKt95vLsj1MWMuXnu3uGZnEnFRvAnEnWiVHj6JuBKoCwaD+9q2XVRYWPiyO4HY+berrdK7o0x8XTs7HqgBvnQ2MPmDfQiNm47ETI7dG7iCyHFTZfTOOildLdy72zECmMr/qrD7ijDj1oOBI5z97QF2OlXq63HGTZqILCKSXEqgi4gkX5DEPiAH6SaD4wzJ1WBQj64q6KKkf72dNfCiAzvtrYHXK8augkQGdmoty9pJ7DXw3H9jJvSCwWBJXV3d0Xl5eZsKCgo2JPuau6lkf1+318I9WXZj1vBe4dweAVwMfIpJ7H4bmIG5vm1EtjBMR7V6Z3TXBHq0IOY9DH8fe2GCQ+OBic7+RwNf+ny+9zGdXdykepXWBRQR6fESmRzlTj7MxXEDaMwkSWLbdr5bGR6nVbrbScu9rxgzwTFarFbpEVXhUWOnxrCOAhHq6ur6AAMLCwvXpOzCu5dUfF/H22cyf2/4MeOfDcBS574fOf9uB74FTHJu1xK5jrqPxH7XZ0K6Euhd1eBs253bHqA30A84ATPZfwCQ5/P5XsNMRHaXKvNpIrKISOcpgS4iknwdqUCPNSDNFbkcDMq163Yrh9p+Utut0uOtgVdM7PeznsjAzvawwE50UEet0rsm2W9cJr8QbnDhE0wbQwsYRKh94WjMrH8wlRnhwaGtmIqB9qQjUEMajpGK62gEdjgbmGDQ9zDvdSFmMoMfM/Fhq9P63Q0ObfZ6vcnqEiAiIumRcAU6OTpuchKPufghVS3c23pSqFV63EnEMdYNL4ixKz+hScS1bbRKDx87qaNW95Dpn40A5jP5P5zbRUS2fZ+O+Z4LYDqAhY+bahPYv8ZM8QUx7/3usPv+P8xYtRw41DlmHVDp8/m+QRORRUQ6RQl0EZHkS3Swn8stzCE3E8munLhup1V6cTAY7A/g9/vH+v3+IK1bpYcHdmJ9NmnGqQonrFV6nKBOrVqlZ4W2BvSpDKZEV2/bhBK6Hzv3lQDDCLUwPB7zfesnFBzagAlOxFqrNV0t3FMtXUEtMGvUr3T+X4CpUv8WsL9zHnVAtc/nWwusJhQcqlBwSESkW0ukI1euJlJduTxmghy4drdVujM2KggGg/0aGxsnxFpaqhOt0qsIddeK1VErG1uli5GOz+rh+28AvnE2MJOehhC5jvpU57FKIhPq22Oca0+pDk/kGOkYjwQxcZO1YcctxYybjsZMRHYnPWz1+XwrCU1E3uL1elO5nr2ISI+lBLqISPKphXticjUY1COTOXFapUdUO9BOq3S/338KrVul11mWtQtT8dChVukiGVIHfO1sYKqk9yYUHDoUmOY8tpPI4FBFGs6vp7RwT/QYRB2nGfO+7gx7TgkmOHQk5r0PYqrWtzvrApYTWhewPsXnLCIiiWu3c0vYpMhcHDeAxkw9jtMqvcS27dI2WqVHTyL2hL1+r+bm5n0xnXnCk9/RrdJbumu11ypduoVs+zlu73qCmPbtPuBD574+RFapH4T53m/EfFZ3x0ybUnC+sXTHNdCTdRx3nfQ9mEneYCYil2He+/HOa+oxE5HXoInIIiKtKIEuIpJ8auGemFwOBmX0up1W6gXBYLBVUCdGYMdNjnemVXqdZVl1wWCwzO/3n9arV68/5Ofn+3KgVXouDjTTfc3pqkBPRADY7GzvO/f1JVSh7ibVLUxlRh6mvd425zXJrvzJ9gR6NBvTBrIWE6AD8x73wUxsGOPspxHY7fP51mMmP7jBoW1er1cdK0REMqMjFei5Om7K+NghQ7pF54E4rdJbtUunY63S6yzL2mNZ1vbwynDLsuoaGxtPsSxrW0lJyUuWZSWyNJD0LNk2Tuzo9ewGVjgbmJ+VfQiNm6ZgWr+77ccLgIMxSfWqrp9uK9mcQI+lGdPZa1fYfe5E5COInIi8I8ZE5Fjd1UREspoS6CIiyZdolUSuBkNcuXz9Sb1u27Y90UGdYDBYSudbpbuBnWrLsnxtrH9Xn0ir9Obm5hF+vx/LsppzIHkuqdcTvomqne0L53YhMJTQeoCjgXGY34NbiaxSr+7isdOZQE+1zl5LANMasjLsvmJMcOgQTIDIrcrY6QSH1hMKDtV04ZxFRCRxAdTCvT25PGYimUuehbdKj9NVyx0vlYaNmTraKj16aak6j8dTByQ0DmpqavJblhVQ8lySoCcsG9WMSdCWh+1zIGbMNBnYCzjLeWwPkWMmHwl0MWlHriXQY6lztq3O7TxMlfpgYF8iJyKX03oisn5XiUhWUwJdRCT5Eq1AVwv3HLx+p61e3Otup1V6vDXwesXYVbxW6S2t/tLYKj3bZtpL9kpV8rkJWOdsxwKvYgJFbvvCMZj242AqM8KDQ1sJTczqiGwIBiXz61HvbNuc2x5McKg/cCKwH6bK5W2fz7cJ+IpQcMjn9Xq1lISISPIlOmZKaiK1h8nJMRMJTJwIa5XuJsRLoycOt9UqPYzbKt3d4rVKd8dMDSlsla5xUxZL0WTyTH7PJPvYNmbJqwpgACbO8QdgGKHOXsdh8hkBTGvyDYTavtd28pip1N0T6NECmGr/8Ir/IsxE5IOAY4AJwHtAuc/n+4rIici7k3AOIiLdhhLoIiLJ5ybQ2+QMunO1FSHkQDAorFV6S1V4MBgcDBTW19cfR/zATqz3pYHI5HerVulhQZ1aZw28tF2rdNyyZcv69u/fv3nMmDE9thWa8z2ebO0N/tMVhEgV9/y3O9ty5/5STHDITaqfiKkA8GNavW/EBIg2YRLBbe0/XXpSAj1akFCnADAJ9FLM+z0O03bfbR9Z6fP5vgHWEkqqV2ldQBGRLmt33BSWqMzVcVPWj5mgdav0YDBYBhAIBMbX19cPDa8Kb6dVegCojdEqvQ5oqQoPGzvVq9q7e2ra2FRU4C1otPJDkxWaK5oLrALLzu+b3xMnNib757i9/fWECvT29m9jxj3fOBuYvwV7ExozHYJpPQ6mNXn4ROQdtD+uTIeelECPpcHZtgNDgFOd232A4zFj1mZMlfpmYBWhSQ0+r9eb7CXLRETSRgl0EZHkUwv3xPS464/VKj3WOni03yqdQCBwSFhgZ7dlWVvDK8M70yq9B+hRX+9U2r59e+GLLy4fOWhQQd1PfjJydX5+fk9PxCX7/DP1vZKu9uex9l+LCTascm7nAV5CwaGJwFHOYxVEBocqwvaTa2ugJ4sH8/d7h7O5xy/FBIeOBmZgAvO7ga0+n28loeDQFq/X25iG8xQRySaJfL5VC/cedu1Oq/Re7gTiNlqlt3TXInardILB4BjLsmpwkuKxWqW7YyaPx1NLgq3Su7kefwFdFawLeipfrBxWsHdBXf/v9vdZ+ZbdXNFcUPlC5fD8gfkNA743YEumz7GTevqYL1omxkxBTOX5FuAD574+hCrU3aS623o8fMy0GdMZzKUW7h3nTmbbg2mj7yrEfB3GYN5/MOPbqhgTkSs1EVlEegol0EVEkk8t3BOT0WCQ0yq9V7ygTlRgp5T4rdLdNfBqid0qPaIyvKmp6bBAIDCpd+/e96bxcjNNg6Mo7777/oDt2/uWVlfXFn3yySd9jjjiiK6ue50pufw7rLMSfc8ChIIM7zn39cMEhdwA0QRnf/XO8zYQCgplQzAonQl0i9aJHHed9D2YIB2YarcyzPs/ntD7X+3z+dYAqwl93SoUHBIRaVMivyMTnZycrTKeQI/RKj1ijNSFVum7PB7PpugJxM7WWF9f//OCgoJ/9erV6/N0Xm+G6XMD4CnxBPuc2MdX/Ur1PpWLKr1lx5RVVP69cpiVbwX7nNhne6bPT4D0VKAnajfwhbOBSebuQyih/m3MRFgbs5yTm1DPJzvGTOk+DjGO1USo/b77PHci8lHAdEITkbdFTUTerInIItJdKYEuIpJ8CbVwd56Tq60IAexkrmVo23ZBjMBOaYwKh460SnfXwCuP0yrdXQMv4fPM8RaBuRr4jLB9+/bCjz7aMmTAgBn1tbVbC5cu/XzwxIkTd2dBFXoyZaqFeyYr0BPhrkf3H+d2L2AooeDQUYQm+pyDWXPdDRAlez26bEygJ3KcZkx7yF1h95VggkNHYFpIBoEaYIezLmA5oeBQj12yQUQkBRIZN6mFexI/QzuTiKMT4bEmEpeGjZk63Co9ammpDrdKt207LykX3DNpzAQU71dcC2ypfrl66M6nd/b2lHqa+3+v/6Ye2r49E9LxWT3VLeI7u/8mzLrc68P2NYjQmGk05nM7wEDM+MkdM23F/H5LllxJoEdrayLyMMzyWeETkdcSORF5hyYii0h3oAS6iEjyqYV7YuJefzut0ltVPBC/VXozTgCH1q3Sw9cTT2er9Fz8umvgE+bdd98fsGNH/5L99jtgZ0PD0MY1a74Z0MOr0CVzGjHt8NY6ty3M2t1nYBK844DJzmPVRLYw3EZirXPjydUEeix1zrbVuZ2HCQ4NBvYl1EJyt8/nWw98TSg4tN3r9ebyxCoRyW1q4d6+tsZM4a3SWy0rFWsiMVAc5xj14clvy7K2EKdVuvP/phS3Ss/Vr7vGTWHyB+a3tNv29PIE8krzevJnplR9L2fqe6Yn/WzahJZp+ti5rwT4ESbZ3huzlnc+4MckfDdiunttwnzO76x0vU/dLYEeS1sTkScBUwlNRK6IMRG5trMnLSLSWUqgi4gkn1q4hwlrlR5dFV4YDAZH19XV9aF1YCfWGnhuq3Q3sFPbRqt099/m9F1ph2T9112AGF9nt/q8f/8ZDR5PHiUlA/wwliyoQk/mebf389GTK9BTvX8bU6EO8L+Y4EQpoWqL4cD+mARvM2YdwPCkekMHjqUEenwBQt0CXEWY4NDBwCWYr8HbwE6fz7cKUyGzCdjk9XprknQeIiLdXbtjJidJmxMTUGO1Sg8EAiMAT319/Xeiu2vR8VbpEROIw8ZMDZZlddfPoFn/dZfYX2N3zfO8srymkgkllTVLawZXLqr0umuip/skkyRbxkyu7lqBnog6zO/KLcD/YD6b701ozHQoprMUwE4ix0wVHTi3bK1AT1bhSbyJyHthlis7EngL2OLz+cqJnIi8TRORRSTVlEAXEUm+IFncwj1Oq/SIVn8Jtkq3bdseBBDWKr1VZXhYYKexGwd2OiIbrqGzcj4A9u677w/YsqWorH//+vrNmz8qA7CsAnvNGrunVqHn/Ne0g9KREI4+Ri3wlbOB+fzvJRQcOgw42nlsB5HBoZ3tHCebEuieFB+nwdm2A4c7x6vErGt/AqFJDbt9Pt8mYBWhdQF9Xq9X7UpFJBslPG6ybbtHjZucScTFzngpoio8VoU4ZsJbrFbpQcAKBoMjaLtVevj/syGhoDFTboj5dQ7WBT2VL1QOt/KtoNu23dPb469+pXqfqn9VDek/s//WWK+TtErHGujpXJ88gJlcvBl437mvLzCCyKS6hflMvwlTob7ReU284o1sS6C7f4tTdazwicg2cBJm/GQDB2Eq1d328JVOlfp6QhORk71smYjkOCXQRUSSL9EK9IxXUiTYKj16DbxYfzv8hKrDa+O0So8I7NTW1l6Vl5e3sri4+LV0XnM3kPGvewbkcgAsQnNz0DNuXPNOWBZxv2WB3+/vUYHhLJXOhG2qxbsGP6EEuas/ocDQCExSHcyadO5zN2AqNNxEbrYl0C2SV0mR6LF2E7k2fSGmSn0sJkBnYyoyKn0+3zeYVv1uxUWV1gUUkSyQ6O+xjH5+jtMqPbq7ViKt0iGyo1YdsDUsAR49gbiuqanpYL/ff1ppaelD6brebkIt3HOYp8QTLJlYsqtov6I97prn7pronl6edH1ek/Zlw/drW9dQDfzH2cB8Vh9KaNw0DbN+uo2poA6fiOxOjM+2BHq6x2dgOgVUYBLpYJL4vTHjpuOJnIi8mdYTkbtrZ0oR6QGUQBcRSb6EBvvOWttJCwg4gZ2ieEGdOIGdRFql11mWVUWobXpSWqU71eS5FhBx5ep157xzzvnuFkwSUjovXcGBVMhEBXoiKp3tc+d2L2AYoeDQ0ZiAURDwYQISlvO8VMrmBHqsa2rCBIcqwp5XigkOHQ3MwFRl7Aa2+ny+lYQS6pu9Xm9jis9bRCTZEv39nuxxU6tW6XHGS+GTiGNNdGwicowU3So9fAJxbWdapbvPt23bbWefE8KuNXcuWiL0PrJ3VfR9ThK9p0r393Kqx0zZUoHeEU3AOmdzX78XoTHTGEzLcTCf1zdiJlN5nC2VY41sTqBHv29tTUQeAxzi3FcLVMWYiFypicgikigl0EVEki8pLdzbaZXekhyn/VbpjZiqcDeAE90qvaXiIc2t0nM1gZ7LH9Rz8eudK5L9fZ2pn5OevgZ6+DG6ohFY42zuPgcTamE4HvP363hMGz23Qn0jpjIgWcGhVLcIDJfOiRkeQpX8bXHbE+4hNPGmALMuoPt1sDCdAqp9Pt9aYDWh4NAOBYdEpJtL9O9F3HFTVKv0VhOI4ywvVRhjVwFaTyKusCyrlsy2Sg+fnK3f6blBY6bslW0/wz15DfRkHMPGjH22A8ud+0oJTUQe4fzfAm7AtHp3x02bMJ/hkyXVy1G50r3EVqLHamsi8lHAdEITkbc5E5HdKvUtXq+3IXmnLSLZRAl0EZHkc1u4twR5bNv2RAV2SgOBwAiA+vr6U+IEdmKtgRfeKr3Osqya9lqlO5Xu3VEuJ9Bz6rqzZO16iS1V38uJLIEhsaUiqGED25zNXX/gFkygqBkTIDoA83evGROIcNsXbsKsE9gZ6a5wSNf3VVeq3ZuBXc7mKsEEhyYBU4FgQ0PDnsmTJ0/esWPHBNu21bZQRLqriN+78VqlA55AIDC2rq5uEGFV4c6/nWmVHj2BuM6yrKZuWuGdywn0nBs3kXtfY+kTBijGAAAgAElEQVS6eN8zPb0CPR1S8R7VYlqIr3Jun4hZu/t9zJhpIiahCybZG972vYLOS3cFejrijF0ZC8aaiJyPGTMNA/Zz9l9/8803D3/qqacebWxsfKKL5ysiWUgJdBGR5As2Nzfv09TU9G3btguI3yrdPDkYHE3rVumtgjoej6cOaO6mgZ3OyMWAiCtXr1skEZn8+cimCvR0VGz4gI+d2/nAPoRaGE4CjnEe205kcGgXicnmBHoyj1XnbFud23lNTU39duzYcQDpa0svItIZwfr6+inBYPDosEnEebGeaNv23kAvZ4xUaVlWvFbpdZZl1WfRBM5cXQvclYvXnYvXLB3XHb5Pcr0CPVHNwHvOBtCPUGev4cAEQl2lNhHq7LXFeW0i0rUcVXdo4d5ZfmJMRF63bt1BTU1N3iQdQ0SyjBLoIiLJZ1uW1eTxeKqAnfECO01NTeP8fv93S0pKHk5T+7/uJlcT6NkSzOuMXPx6S+dk+89JupbJSOcx/Jhgz4aw+wYQCgwNBw537q8jMqG+hdgtzdPdIjCdLdxTeaxAc3OzuyagEugi0p3ZztioOrxa3Fkv3P1/fW1t7Zz8/Pz3i4qK3s70CWdALifQs/3zYCy5eM25Itt+htNxPdkwZoqVpK9yts+d272AoYTGTEdjlhoJYibIho+bdhNbuiYDpHOJrXQcq66hoaEB095dRKQVJdBFRJKsrKzMrqmp2Z6fn78EU50XU1hVRLYNpBKVywn0XLtuBYKyXzord1N5vHTtP5XSeYz23id3hv9nzu0iQmsCDgeOxSxXEsQk0cODQ3s6cJxkSFfVRlqO1dzc7AFs27b1+1dEujO7qKjoc2BFe88j9z4/u3J5zGjbtp2L1y3ZK52fy9KRUO3pLeLTdYz23qdGYK2zua8ZTGjMNA6Y7Dy2m1CF+kbMElvBBI+TDJmoQE/psQKBQDrHgSLSwyiBLiKSGu4H2PaeQwLP+//Ze/PwNq7r7v97ByAAgvtOkRRJUVJka7HlJY4sy7sdx8ruxkmaNHGbOomfJE3TtKnb/N7USev2Tds0S5vtTVw3bhoncuSkXhJvUbzbsi1biy1ZlmRxEUlxETcQBEAsc39/zB0NSIEECGIuZjmf55lH5AicO5czAOec7z3f41TcnAhz5bxdlABzk2DllmtaKJxi4Z7vGDEAx8QGaFUFenKoHVof9YvF/00CGBVf1wAYyGO8pWBnC/czSCQSjDFGiSCCIKxOrp9TucRWToUD2nO0g1p5LQU3TtqNcybsh9n3qVMs3PP5PXFowvgwgD1iXzmMhcjtAM6G1vIkAS1OqoYmrgegxVxmYWcL94yoqspAFegEQSwACegEQRDmkEuSR0+GKC5NhrhVQHeTuKpjuzknk0l2110/7bjwwvPHzj333Olin48Lyenz08RxnVCtYYcxdFvCIQAviX0VMKotusS+GwC8C1pPQL3aoh9atUahkCmgm24XLyrQKRFEEITVyfWzkHPOlewvcySurkAv9gkUAVvMmascoV2hhopLK8aUgHJa3Jp+dromsDowU9JcEi/m+RGnsXNMI+P4dhLpwwAOiw3QNJ0VMOKmCmgi+q3QFiGnO3uNFWB8HdkttkwfS1SgU9xEEERGSEAnCIIwh6wPeGTh7k5LPnHdXTdvu7F79+6aZ5+dXjk29nTppk2bXlcUt+ZsLYfd3zsyz98OAnompgEcEtsKAJ8CcD+AMmjJoYugWb8DwAjmWhhOLGNc2RXoplZSJJNJqkAnCMIO5Cygw/7PAHnh8pjRtdfd6iSGEv6ZPTMNsz2z5fUfq+9VAooaeiJUN/309Ao1og5XNVeNZj+K6zDrXl7oc1RGhbidj69jFwF9PkkYMRAA3AQgDq0lSju0uOl88X/RtNf2QWudlcxzXClV4fPGMvUaUQU6QRCLQQI6QRCEOaRAFu7ZcGtCxK3zBmwy72QyyZ544pUVs7Nt3iNH+uv27NlTfdFFF00W+7wsTiGDWplCZqaxYeL4MqvDzURW5YE+zkloVeo6dTCqLToBXCj2z2ButcUgck+GOM3CXQH18iMIwvqQhXt2SEB3H5afs6/FN1t7Y23P+C/HO0/99FSHv8MfDr8QbgpuDp6qupbE80UodMxUbOzeXskqPdALNc4sgANiAwA/DNv3lQAuBeCD9jf1JObGTbm67jmuB7qqquTcRRDEgpCAThAEYQ45W7jDsCVyGxzunbsVgl2Z2MKKUGf37t01x48rtatWXTs+MPBs5RNPvLjiwgsvnKQq9IzIvpdlBux2xSkiffo48+cyJrZ94vtSzE0OXQGgBFoiZBBzk0MzC4xluq26zLFEBTolggiCsDpLqUB364OYmwV0wH3zts0zbmBNIFJ7Y23P2M/HuhJDibLSDaXjNe+uGcr+k4QkzBZunfDetJOFez7jzAJ4U2z6a5pgxExnAdgi/m8KRoX6CWhOX5kWuUmxVZ83lqmLgsnCnSCIxSABnSAIwhxyecDTe6A7IfDIB7dWFNgmKeJG9OpzYD1KS+uSK1a8bfrIkSNUhe4enFSBbvcxljJOFMBRsQFaskVPDrUD2Ahgq/i/Ccy1MByF8fdIVsW26WNRBTpBEDYhp88pl7dAcrOATnGTxYn3x0v1r5NjSb8aU5X0nuiE43FCBboTxsh1HA7N1WsIwEtiXwXmLkReDy2WSgDohxE39QOIwZkV6CSgEwSxICSgEwRBmEMuFehut3B3qxWjvnACjLlu+pafsF593tLythAAlJU1JVKptzCqQrccdk2oykw4OGGMfH9fui3hSQAvin2VMBJDKwFsgmF12A+gHFoCyQetf6CZmJ5Iowp0giBsQq6fhW6NGwARM7p00bVbF07YYs56z/Pg5uCp0rNLQ7qdu94TvdjnZ0GKcV3tXIEuS3h2yhj5/r6mAbwuNkDTilpgxEwXArhM/N8IgIj4ugbaQmQzIQGdIIiiQwI6QRCEOeQS7JOFu02SAyZRzD7PsrHNPJ96am9zKBQMpFJPnn5fJhIRz9Gjat3evXurLrjggqlinp+Fkdk7WsbxbXPPZsCNFei5EAJwUGyAZvHeAq1CfSWAIICzoVkZjmCuhWGh3SdMt3AXVoSUvCYIwuqQhXt23FyBDrhv3rZ4Bo0PxP26eK7btus90UO/CzVUv6t6uNjnaFHMuL4LHVPGe8cW9+siOK0HeiHGSUKLgfrS9tXCcPZaK/Z9BpqYnt4qa1D8fKGQYuEuBHSKmwiCyAgJ6ARBEOaQy+pFtydD3Cqg2z3IXA6Wv94bN3aMd3RMT2stwAwYq0d9ff1skU7LsphUDWX5+2QZOKU/uR0F9PkkAPSKDQA+C60SvQ9agqgTWsUFAIQxNzl0EsurUpBRga4wxigRRBCE1cn1c8qtcQPg7pjRdXGTcCiz/LX2tfpm6/6w7nhgTUCvRkVgTSBS95G64yUrSpYaM1l+vlbEAq4UTqhAd8oYZo8zLrb9AM4D8B4AP4NRpX45tMXJKjQRPT1uCi9jXKpAJwii6JCAThAEYQ5ZbQbTEtvFDnyKhVsTYelJMLckhWwzz/e85z1DBTycG+/vQpGtksKuPcqdUh3uBAE901gRAHvFBmhV6ek9Aa+CFj+loCWH9Ar1fgAzSxzLVHE7mUxSIoggCDuQazyggirQ3fhc6cZ40TZxU7p4ruPv8MeKcS5ERszON9j9+E4aQ/Y4AHBMbID297kRhrPXegAXi/+bhCGm90Fz+sr1PKV8/quqqoDiJoIgFoAEdIIgCHPIpU+f3gvbzckgtyVE0nHz3E1hYmLCqygKqqqqCmkbRhSHYr8/bJO8zAAJ6PmRyVY9AuCI2PTXNMMQ1M8BcIn4v3HMrbYYzXA8HSkV6KBEEEEQ1kdFbp+H3ALVlkWBMeZmAR1w77wLDk9yxrzsjPcbVzmYUrRfs52fuZeKE+9lu18/WQK6DGQJ6JliJhXAkNheFPsqYMRMKwFsED8bh7b4WI+Z+gEs5FohxVqdKtAJglgMEtAJgiDMIWcBPYfXORKRDLLV3F999dWKXbt2rfj85z9/RFHyXvdg9yBzOZh2vZPJJPvGN76/wefzqrfdduurZo1DLIjs+9rsCnGzkCkIO0lAl2FFnkuCRrclHATwgthXhbnJoXPEsWYxV1AfgJYwAiT1QCcLd4IgbECu8YDt4oYCYpuYMTGS8MX744Gy88tC+j6e5Gz6menayisqx/I4pFvjpoJfazWusqFvDp1duqF0XO9ZDgCjPxntUGfUkqbPNh1b7OeJguGke9oJFu6QMIbMynBZMVMu85kGcEhsgGbx3gIjZroImvU7oFWl685eJwBMiP2mx0wACegEQSwOCegEQRDmkMuDK1m422juqqriu9/90aYDB8ba3/KWh6e2b98+nOehbJMEKyCmBz3PPvts7eHDqSaPJ8F37959YsuWLZNmj0mYSjYLd7OQJQxTD3RrjaOPlc84U2J7TXzvg5Yc0i0MLwZwpTj2MLSkkA9aEsk0qAKdIAibkOvnrust3O1QgX/yGyc3RV+Lrmj9ausz5ReVT/IkZyf++sSFkVcjrf5O/yP+Tn90iYe0VbxYIEx55lF8Cve1+MJTj0x1AkDNu2uGRn8y2jGzZ2ZF9fbqbjPGJIqODOHW7seXFTdZOZaRNU4CQK/YdOqgxUvtADoBXCj2z0CLmfTr44GJcQ1ZuBMEsRgkoBMEQZjDUirQ3ZwMsk1C5Omnn67bv3+8IxRqK7vnnvvXveMd7xheRhU6YKO5W51kMskeffTZlanUeiWZjLNHHnlyJQno0jDrPnbq+8Mp4jYJ6AsTB9AjNv249TCqLboAlALYAs3KML1KfQgFSt4kk0nGGKNEEEEQVifXajlbxQ0FxjaLb1tubTnQ++e9lQNfHdjW+netz4zfM742sj/S1vDphhfzEM91LD9vu1D/8fo+AJh6ZKpTF9Krt1d3V2+vzndhOGENilXVLmths9ljOKkC3W7jjIltn/i+FEAb5rp7eQD8LTQHsPS4aaZA56AvUKO4iSCIjLhVtCEIgjAbsnDPjm0SYaqq4u67d66LRtf5m5o+NXbo0Ezbww8/3JTn4Zxk27ZUTLneWvV5srGp6bJQY+NloYMHo827d++uNmMsQgqL3SdmC6p2P77sMczGjgL6fDi0fuivALgPwHehJXxeBXAAQDmAqwHcDOBvAPwJgGsAvAVAMN9BU6mUAjk2jgRBEMsh13jINnGDCdgmZvTWeRMd3+l4xlvtjfR/uf+qyL7IyoZPN7xY98G6gTwP6da4ybRrrYvoAOCt9UZJPJeHCS4S2Y5n9wp0p4jbdhS2s41jVowRBXAUwO8B3AXgcWiLkx8DEAKwEcCHAPwVgM8DeD+0qvVGLONzkyrQCYJYDKpAJwiCMIesD5RpvUktnwwxCdskwvTq8+rqT02Xl2+anZg4p3IZVeinbRgZs8X0C4FpgVx69Xl5+co4AIyMvKWSqtDlwhgr9DV2esLU7vOjCvTljzUC4BnxvQdAM4xKi3MBXCL+bwxzqy1O5XKeqVSKKikIgrADKnL77HW9hTtsEjd5qjxJT5UnlhhKAAD8nf7lVAlyO1jXFxhTY+TRn4x2AAArYankeLJ04oGJ5vSe6ITp2D0GSEfGe5N6oLt3HB0VwAtiA4BKzK1Q3yTOaRZAP4yYqR+a+J59AOqBThDEIpCAThAEYQ4p5FiBzjl3bTLIDgmR9Orz+voLpwCgpub9oUOHvtb28MMPNy2jF7rl514ozFwooFef19RsnpmdnfIAQHX1eTMHDx5ppl7ojsauFeIyLdxl4CQBXZE0DnBm4ikFYEBsu8W+asxNDp0rfi6GuYL6ALSegnMQAjpVoBMEYXU4DMFwsc9g2yy8NQHbCOh6z/PYkVhT3UfrXgk9Hlo18NWBbXpP9DwPa/l5m4Apc07veV69vXr41H+fak/viW7GmITjsbvjlazPFycJ27JjpvnxTAjAQbEBQAmAVhgx0xYAV0A7xxFo8VKf+Dfj3yGRl6S4iSCIjJCAThAEYQ65JHlskwwxCVskwp555pna/fvHO4C1GB29t0rfPz1dV55PFboJlbp2ouDX+5VX9tUxxvnU1N2lU1Mo1fd7PIy/8sordUUS0N10jWW/hy3/mZEFWfbqThG2ZVegy0qc5JJ4mhTbq+J7H4zkUDu0CnW/OM4QhKCuquoJRVGmkskkWRESBGEHyMI9O7aJGQf+fuAcved53QfrBmreVzPQ++e92wa+OrCt8/udu/Log+6mZ2odU+asxlWWHEkG03ue63buiYFEmRljEkXH7JhARg90J1SHO60yXGYFei4xUwJAj9gA7fzqYAjqq6DZvANAGGkLkVVVPakoSoos3AmCWAwS0AmCIMwhl4cvsnC3wdyj0ai3udk7kUq9rAAvn97f2IgpRQnyfC3cYYO5FxDTAqyPfvQPey677ERGF4COjo6IWeMSjkWmYGsWJKDnP1YxqymyEQfQLTb9GA0wkkNrAFy0Z88e3HLLLXzdunXRhoaGacbYhQD2c87PqFInCIKwADlZuIvWV6507UpbfGv52KHmhpru4DnB0doP1A4CRk/0sR1j7b5231LFc8Am8aIdUHwKb/6r5teZMvfXWf/x+j6u2vmx11Y48V62+81DArp1x8l3LA6t5dUpAHvFviCANhhx01UAvB/84Ad5NBqNr169mo+Pj1/MGHudcz5aoHMnCMIhkIBOEARhDipyr0B3ZTIINkmIXHfddSPXXXfdwyYc2vJzN4GCz7mxsTHe2NiYU28rwhHYXeCWdf5OEbadLKAvdyzdlnAExuquspKSkvVXX331ua+++mrjwMBAI4CXAEQZYy8C+B/O+R3LHJcgCKKQpFu4Z3udG5+dARstvi3bXBYq21wWSt/nrfMmmj7T9OYyDmv5eZuAKXOeL55n20/YhoWeKakCPTdIQLfmOIUcKwLgiNgAwAOgaevWrVv37du38sUXX6wA8HUAX2eMHQXwHIC/4pyfKsDYBEHYHBLQCYIgzCFrksdO1QQm4dZEmBuvu13FTiJ3CnaNRQ+yYiGrBzpZuFtrHH0sK9kR5sPMeeed99J555330g9+8IO1b7zxxprZ2dk/B7AVmuV73XIOzhj7LwA3LfKSLZzzF5czBkEQrmMpArqbFx0X+/moWLgxXuScU+jkYAp5ca3w3jA7pjG7vZKsPutOimVkt70yY6wUgMEvfvGLOwFg3bp1NwO4BUA5tLjprdB6recNxU0E4RxIQCcIgjCHnC3cOeduTgZZIeiTDWVECCfhxvew1ZGVCAJIQF/OWKYmnlKplMIYS3LOdwPYDeCbBTjs3wP4QYb9D0Kr7HipAGMQBOEucv0sVDnnbs1fuXHxrQ7FTQRhXZzwmUQW7tYdR+ZYCoBBzvkrAH5RoGNS3EQQDsGtAQhBEITZLMXC3QmBRz64XUB33dxdWjlDFBa7V4hTBbo1xwHMqwrPhOnXKJVKMeS2mC9nOOfpPdgBAIyxywHUA/h7KpkjCCIPyMI9O66NHeDe6+7GObsB2ddVRkzghJjGCWPIHMdRMRMAqKpKcRNBEAvi1qpHgiAIs0mBBPRsuDUhAsB1YjIFB8RScfo9Y+dkFuBMAV2mHaEMAV1BgRNBC/Cn0H5vd0oYiyAI56GCBPRsuDZmZIwB7pu305+BCecg471JPdDdO44+lunxmXAFpbiJIIiMkIBOEARhDrlUoOsPgm79LHZrIoySIoQTKfR9vdBng90rxJ3SZ0/WZ7f+99FJySApiwLMqKSYD2OsEsAfAPgd57zPzLEIgnAsuYrDKtwdMwEUN7kJN15rt2DGPb3QMZ1QgW42VIFu3XEASdXuMgR0ipsIwr64NQAhCIIwm6yrJBljbk6G6PN349zdfN3dOGenY8Y1dfJ9IsvC3WycWoEuU0A3uwe66QI6gI8ACAD4T5PHIQjCuegW7rm8zsnPB4vh5tjBzdfdTdA1zo9i/97MHl+mUGsmThO2HdcDXbhDml3pTnETQdgUEtAJgiDMgXqgZ8ftCRE3zd0Jge+ScZlNf6Ep1j0jS7C1e79AHScJ6LLGkTInSRbufwpgDMD/mjwOQRDOhSzcs+D2Rddw37xdGTe5BCfey3aPmcweQ8dJvcllt71yioU7xU0EYVNIQCcIgjAHsnDPjlsTYW5OirjxehP5USwLd7NxkhUhJI5jNjLvKym29GZXoDPGNgG4AMBPOecJs8YhCMLxkIV7dtwsoNv1eW+5uOZapy0QIQqP2TGBjAp0GccnC3drjgM4xMKd4iaCsDduDUAIgiDMRq+mWAwOuLpK1VICejweZ3fccceqoaEhn8lDuTEJRokR5+OUa+yURI2TBHTHiNoCWRbuCufczEqKm6H9vsiGkCCI5ZBLzARYLG6QjCViRq5yzLw8UzV//8wrM5U8adq5ufG6O+WZmsiM066vE1y1nBLPyIybnGbhbnYFOsVNBGFjSEAnCIIwh1wTQYD7kgI6lkqI3H///St+8Ys9595113+vljSkZeZOEMvAzITpcv7fzTitB7pjRO15Y8moQE+acWzGmA/ARwG8wDk/ZMYYBEG4Br0H+qJ/u0SVqlufnS0RM4YeD9UP3j548an/OdWu75t8aLJx8J8Gt4ztGGszcWi3XneCKAR2rkA3G5lOV04S0GX3QLe1hTvFTQRhf7zFPgGCIAiHkvWBkjFGFu4WCbri8Ti7//4n1o6NNVU99dSxrg99qKens7MzasZYLreps8T1JmyNjApxu/fac5qFu2P6kgukVLurqmpmJcX7AdQCuMOk4xME4R7Iwj07lhDQK6+sPBV5JdIzce/EegDw1nljo/85ujmwLjBSe2Ntv0nDujVuopjJmcheeCzjPrJzTOPEllROE9CdYOFOcRNB2By3BiAEQRBmk8sqSUskQ4qIZQT0+++/f0V3N2vs7LxleGysqXLHjns6TRzOjdfdrckvIj+c/N4gAX3p4zhNQJdl4W5mD/RPAJgGsMOk4xME4R7Iwj07logdmMLQ/JfNh8u3lvdM3DuxfvRHo+cH1gVGWm9r3a/4FLP+frrxulPcRORKsd8btLA593GcJGxLqQpPG8vuAjrFTQRhc0hAJwiCMAeycM+OJRIievU55xexYHBVvKLi2uhTTx3r6unpKTV56KLPnciPoaEh37Fjx4LFPg+LUbDAlvNFD2V2okPminozIQF96ePASWOlUikFJlm4c86v45xXcc5nzDg+QRCuIicL9xxf40jS3KuKPn+mMJRuLB3Xvy99S+mEieL56WFNPj5RYNSIqnD1zNtCjaiUgy5wzMQ5X+z9IeM5muKm7DhRQKce6Lkfm+ImgrA5ZOFOEARhDrmsyHS9hXuWgE8KevV5Y+P2KQCor798urv7scYdO+7pvPXWv37dhCE5kFUkdCpFv97LRVVVfOMb39s0NRULfOc7X3s+GAzKWn1tSXJ9D3POvaqqlquqWsY5L+Ocl4t/T38NoIxzXg6gFEACwEYA3WIbhTMSNFSBvvRxHCNqC6RYuHPOGefc1Z9PBEHYAr0CnSzcF8YyAvrkQ5ONo/85ujmwPjDkqfDEJ/534mx4wOv/qL7PpCGd8OyXD0W/1vnCVY7B/zu4yVvjjTV9oekNpmhTGf7u8Jp4f7yy7fa2vczr6pZmiyKe34J6zKSqanrMVAagPP1rAB4A5wI4Di1m6gMwK+l0nVKBbjZOE7ZlW7jLiGdkjUMQhA0hAZ0gCMIc9GqKBWHs9PO6bQPkZVL0SpJ4PM4eeODJNZHIikAo9GoiFHq1FAA4b2ZPPbXfrF7olkmCycJJfd+ffvrpukOHZlckEiXe3/zmNytuvPHGgWKfUzEQFQ+BVCpVAwCpVGplLBarEgJ4Wbo4DqAMgC/DYSKMsRkAM4yxMGNsmDE2o6pqtaqqAfFz10JLDEUA9AAY0k/BpKk5wSpQxmeL0xJOUkRtgTQLd865WVaEBEEQhSLXz11LLLwtEpaIHaafma7Ve5633ta6n3kZH/q3IXXi3on1njJPoub9NSdNGNaN193WcRNTGMovLh8av2f8rOFvD6PpC01vjHx/ZM3MSzOtNe+vOeJG8Zxz7hGCeJBzXjo7O7t5/oJiCGEcQBBnvteTAMKMsRmxDSmKMgNATaVSDdDumY0ALhZfD0IT08th7wp0J8Rl+jiy7ntZcRNVoBME4RpIQCcIgjCHnJNBcJGQOo+iz31mZsbj8YB3dPQNa4u1NSorEQkElOTY2JjPBAFdx63X3baoqooHHvhdZzx+ttfrrVUfffSZzne+850nM1Sh2zIxxDlXVFUNimqHMyrFMbfioQyasA0ASKVS26AJgjNCFA8zxsYVRTkhxPEZxtiMoihh8W+EMZZRQJydnd2oqioHcBDas+pKAKsAdAI4S7zsDwC8CS051ANg/IwDuReqQM9vHEBO5YE0C3fOuSkW7gRBEAWELNyzYwkBPbg5OFV5VeWbDTc3HNdt25v/svnwaPXobPkl5WMmDu3G627rOVdvrx4GgPEd42d1/0l3MwDUvL/mSM37TFlkIR2xkNjHOV/UXSstZgqk/3wikXgvgFhazDTDGBtLj5nS4ybGWDyt+OI0qVSqOhaLncU53w8gCqAWRsx0nhgb0How665e/Shcix9b36dwVjwjux2VYwR0VVX1cUhAJwgiIySgEwRBmEOuSXhVrHZ0I0VPhNXU1CTvvPM/npY8rC3F1QJh6yBbrz5vbLw25PNVp3p6Xqy3ehW6sE7XBfGFEjvpFQ/zSSCt4kFRlEEIkVxRlDDnXE0kEh8qKSnZ6fP5DmZK7iyTJIyEDwDUA/gsNJvCWgAboN1XIRhiejeAqTzHc0qlg1OqHMjCPU9UVWUwqQc6QRBEASEL9+xYQkD3lHtSTZ9pejN9H1MYGj/Z2L3QzxQAN8ZNjphz9fbq4Yn/nVjDZ7kXAKrfU21p8VxYp5fqMVOaMH7aUWueu6GrpX8AACAASURBVNb8fDqHsZB4hjE2pSjKQJogHk4kEhdxzgNlZWV3McYKKdbp98y42F4W338UQDWAMIC3ArgMmkh4AkbMNIDlCYd2rkDXccIYThXQTbdWj8fj+rMFCegEQWSEBHSCIAhzoAr07Lh17pZIgknG9omg9Orzioo1swDg8VzEF6lCNwVR8eBP74enquoZtulpFQ/+DIeJzqt4GM1U8aAoygxjLLHY+aRSqYpEIoGFKiPyZLGgXBcEX4EmovsBdECrtFgFrQcgAExirqA+XaiTKxB2TzY5qWJDHwdOGktUoFMiiCAIq5Prc7Fb4wZA/I5caGUOuPu625rh7w6v4bPc61vpm4r3x6uGvz28Lr0nugyEdXpwsUpxiBgKWtyUyTr9tADOGBtRFKVbj5tErKTHTNFsbcuSyeQ5AEoKKJ5n+2UmoC0qvke8thFGzHQxgCvEa07AiJsGkbtgKeNiOmFhs5NiM30sWf3CTf/dJRIJ/XdHcRNBEBkhAZ0gCMIcSEDPgggwXTl3gRvnbts569Xn5eVnx8LhHh8AVFSsjXZ3725YbhW6qHgI6omddHF8gYoHz7xDpFunzzDGJoR1+umEj0js6IkeWQGvDGYBHBEbAJRCE9TT7QsBYAyGmN4DYGaB48mqQDcTmWM4IeGkjwPJY5n6PlRVlZGFO0EQNkC3cF8UN8cNaaKcK+cPd87b1nMe/u7w6Z7nNe+rOTn528mm9J7oyxHRVVX1pbWbyuauFchwiFnM7Sc+lh4zpcVLM4yx2QK7axXzunIAw2J7QZxLM7SYaRWASwFcDSAOoBdG3DSExT+j7Rw3yboeThTQHWPhnkgkqAKdIIhFIQGdIAjCHHK2cIe77QhtnRzIEzcmwWxfgX7gwIHakpJUKpXaqYRCKNX3l5YifvDg4dobb8QZAjrnvDSRSLRkSuzMq3gI4sz7IYG5FQ9DepVDhoqHWLaKBwey0HyjAA6LDdAWHHTCqLa4QOwfhSGm94ifk4GM5IaTkjSyEjRSbNUFUn53qqoqoEQQQRDWhyzcs+PG2EHHjfGi/Z/p2dye53pP9NiRWNX8lwp3LSWZTDZkEsahxUvp7lol8w8BIJLmrhVijJ1MF8TnuWvRs5EGB3BSbM9B+3xtgREzXQngWgAxaIK6HjeNQN5nklNaazkpNgO0e0WmgG7qe1ZUoHPOuf0/ewmCMAUS0AmCIMyBKtCz49a504O5DfnEJz7x5tvf3n+KMRYUWxljLAggWFtb2xGJRM5Or3oAgGQy+c5kck4BaIwxFoYhjI8uVPGgKEq8CNNcDoW+rxc63lI/M2YAHBQbAFTASAytBXCR2D8ELSnkX2TsQkICeu7jOEbUFkgR61OpFFNVlSrQCYKwOmThnh23C+iEzWj8TONxVVXLkslks+6sFbgqUOa/0l8WjUU3pAvjENbpsVjsM2mHSEGLl063mxLW6afF8LS4KWKzhcQyz3Upz9EqgH6xPQPN8awVhqvXtWJfFEZ1uozndKcI6Gbj1Ap0BUYLN1NIJBKKwxz6CIIoMCSgEwRBmEOuD2BuTwa5bu4ut2Es2JyTySQ7cOBAxebNm0OKkl9BkrBOL9UTO/N742FexUNra+v85yYOIYZzzsOMsUlFUQYAxFKp1JUej+fpkpKS10WleMShFQ92u4+nAbwqNgCogiGorwdQKfZ/EkZyqA+anWEhkPX7cpKALiOhIcVWXeZYoleuEz9zCIJwFnoFejZcGTcI3Bw7uPW6F2TOqemUx1PhSWXblwuc8xJVVc9oNbWAu1ZphkPMYq671gRjbCaVSq3inJf7/f770ty1Cm2d7nTMeB5PQYuB+gA8CU0/aINh+f4O8bprocVSPdDipvECnoOT4iZZsZmsWMYxFu7JZJJiJoIgFoUEdIIgCHNYykpfN9sRujkydtPcCx70PPnkk3W//vXu1R/+cOTotm3bTgfqnHNPWi/xjL3xYFQ8ZLJOT2JuX7xhvcohQ8VDNFPFg6qqvkgkcqWiKMNer/dkoefucgot3E4B2C82QOv9twVaz/RzAGyF9jk9CMO68AQ0i/18kGXhbjZUgW7xsVKplKzFBwRBEMtB74FOFu4L42YBHXDfvAvyfDD91HRt6IlQa/1N9Uf9Hf4YAERei5SP/3K8q/bG2uOlG0rDYiGxHjdlFMPT3LXmW6cDmnW67q41nWadPsddS8RNGatIo9FouaqqJSUlJX2FmLfFKfS9nO14hXyOTsJoffU4tPvhy9DiohoAG8R409BiJj1umlzGmLIs3M3GSe5g+lgyBXRT45lUKsVs5mJBEIRkSEAnCIIwB7Jwz45b504P50tA9MXzpSd2EolE2YED3WvC4ZbaQ4e6N51zzjkpj8ejJ3gCGQ4TE33x9GTOWKa+eOLr+HIqHsLhsOfOO+9a/d73vgvNzc15H4coGlFoCaJfie/rMbd/+qXQVqj3w6i06Ie1Vq07KUnjRAFdmoU7rHVfEgRBZIIs3LPjZgHdrXFTXteac66oqhrknJd7N3orlMNKy6n7T62pfF9lVI2q5dNPTdeXnFdSklqZumZmZqYMZy5K0a3T9TjplKIovRkE8XC+1ulqXGWhR0ONVdurhpliTHPq0amG8ovLx/OpjieKgr6Y+BCAvdDaYLXDsHw/R/z/JIyYqRuawG4VnBTPyBbQZb1PTe+3nkgkFFDMRBDEIpCAThAEYQ45W7gLm1U34tZEGAdO2+u6DQbMsU7PVPFQBqB8XsXDnOeVI0eO8JERX+Lssy+PHTv2u5I333xz5KyzzhpIq3iYXykuLSB69NFHm556arKzpeUVvPvd2910jWUnOM0ab/41OyW2PeL7RhiC+tsAXA5NcD8Bo9JiAAv/DZBVge6EXn76OE4T0KVYLKZSKYCSQQRBWB+ycM9C2sJON87ftdddh3PuVVW1fH6rqQXctQzrdB9QdkMZZn4/k5x6cGoWKmZLmktmKq6u6Gdelh4npX8dM9s6PXooWjH1u6mO+GC8rOHmhuMAEHkpUhL5TWQVGFB1bdWoqSdQfJyyKGT+jTIL4KjYAG1ReyeMuGmz2D8OI2bqBjCTZQyn9EB3moDuKAt36oFOEMRikIBOEARhDmThnh23J0QcNXdR8ZDRApBzXgkA8Xj82ng8fj2AMpw5/xQM6/QwY2xEJHJm0vrhzSSTyZl779215sSJzTUlJWsnBgZeq7v//t+Pbtq06TWv11vUhEQ4HPY8/viBlbFYV8n+/SPYuLE7cNZZZxXzlGRgxn282DFlvG8Wu49GxPaiOJcmGJUW2wBcBa0iow9GpcUQDLFUVnLDKUkaWQkaKVXhAim/O1VVqQKdIAg7QBbuueHmuMlR8xbuWoH0VlP61wDKVFVdBSAYDoc/Dy1m8mU4TGSeu9ZwJnctpUyZUdqV8ol9E6sAoOq6qjcCFYGIvNmeSdnmslDq/ak3J349sXr0jlEoq5RgdH/UV3FpRb8LxPOCIu6lxV4i4zl6oePHABwWG6C1TuuEETddIPaPwhDTe6C5genIigOcEM/IdtNyjIU7VaATBJENEtAJgiDMIecKdDgsKbAE3Dp326w655yXLKHiIZN1+iy0xE4EABhjYx6P50Sm3niMsdlcKh6efPLJ+u5uT11r68VTANDcfHHo6NHX63fv3l2T3gu9GDz66KNNJ04Ea9as+eDo1NRv8dRTz9W4QEAvOFzLBBXrfbKUzyQOTRwfAvA8tGTCChjJocsBXAPtfdALLSmUsf9jgZFZFeCEhJM+DiSNJcvCnZJBBEHYAb0CPZs47ta4Qcet87dF3JRmnV6WLowv4K5VBsAz7xAqhBgO7b2Q9Hg8hxZw14rkWi0ZeS1SPvmbyY6S+pIIT3Bl7O6x1ek90YtF5eWVYwAw8auJ1ewUKyvdWhqvvaJ2sJjnJAknvYeXOpcINLv3Q+L7chgx0xoAbxX7h2GI6Wb/vmReDycJ6DJjTdPFeqpAJwgiGySgEwRBmAP1QM+OW+detD6GYpW6bp1+hjAOTQw/XQEBoGT+ITC34iHEGDuZoeJBF8WTAKCqqi8Sifyt1+vd6/f7D+Z7/vF4nD3++CutiUSnJx6f8sTjUx4AiMVaSnbteqF1y5YtE8WqQterz/3+y5I+X6VaVbUZe/furHnttdfKN27cGC7GOTkUs5MDy0kIqNDs2wcAPAst4G+FUWlxFYxn73dDszjsgVbRXkhkfLY4rQJdiq26zLHIwp0gCJuQ03Ox6LXsxrhBx61tv4p23YV1eiZBvHy+MA6tunY+cRj9xGcURRnUv9fF8LSvT1unR6PRd6mquqK0tPR3yzn/yGuR8vEd46tLVpSEG25uOK5GVc/oj0fXnLrr1NqGP2k46lvpK6qInhxJnl58rYZUxlWO9J7oxJJY6FnZ9OrdRcbORhjAa2IDgEoYMdPZALaI/aXQnme7oTl8xfM90Qw4KZ5xqoAuw8KdFh0TBLEoJKATBEGYA1m4Z8ftibCCzH2edXrGSnEIYRyaKD7/fkvBsP+bYYydUhSlZ37FgxDEIyKBWRT6+/sD4XDMV1X1ZnRm5s3TdobV1YhMTiYCw8PDvtbW1tlinJtefd7RsW0MgFJT04FDh1p9jzzy+9aNGze+UYxzkkwh7wunfC6o0HqjnwDwFLTn7i0Arobm2HAdtPdjBHN7AY4tc1wnJWmcWIEuy8KdkkEEQdiBXF1nVACMc57eE9xNuDJuEte6UDETOOf+9FZTqqqejpPmC+PIbJ0eFQuJ9bhpNH0Rsd52SsRNiUKcd74oQSXl7/JP1X20rlfxKVzxKcmGTzYcG79nfCXzF7facvyX463Tz0y3Vlxa0c9WsbbwE+Gq0TtGuxpubjjuAhHdKTFToccOAdgvNgCoAfAJaFbwmwBshfa7G4ARM52A1jorX5wUz5CAnidUgU4QRDZIQCcIgjAHFTkEFW6upnDx3LMGAMI6PdeKh9IMh9Ct0/WtX68az1DxkJN1eoFY1kBdXV3Rr3zlM/sSicQZx/H7/WpNTY0Me+wziMViyuOPH1gZiTT6+/p+W8sY8yQS5YjHufLKK+Mtb7zxRv+6detminFuEjDj5immXaeZQXoSRrX5Dmjv05Uw7Au3i/HDMPqn9wCYyGMspyRpnCigy7JwB0hAJwjC+uTaAz29Ut0Wtt4FhuKmTP/JOeOcB0XctFCrqXR3rYWs03V3rXFFUfR2U+mtpvR/ZYksy77Wga5ANNAV6Enf563yJhs/2di93GMvh5mXZ6p08bz2A7WD0Wh0lit8dvqu6fqph6dmqrdXDxfz/IglY9bn8QS0WOkogEcB1MGoUD8fwDZo799+GDFTP/Jrl+WEeEa2gC7rs1AxeyyqQCcIIhskoBMEQZgDWbhnxzVzFxUPAVVVy1Op1AoASCaT65PJ5BqIHuLz+uJlqniIpFU8hBljw4tUPBRFSF6IQlat19XVFbWaYyE2bWod6eqKTgB9UBTFu3Jl21vHxxODkUhDfzAYpIBsaSz2uSAzOWAG6eefAHBcbID2vm+HkRzaKF4/BaPSokd8n20MpyRpnCigS7FwF1a/VE1BEISlqaio4NPT0ySgZ8dNcZNHVVW91VQQgCcWi21bwF0riDN/LwnMddcaEvFRuiCuu2vFiumutQBWO5+CUnZB2RQYDpedXxbS9/nf4k8F/iTweuDsgNPbXsl+D5v5eSmrZZR+/mNi2yO+b4AWM60CcBGAy6GJ5ydgxE2DWFwYlRnPmA1VoOdJMpmkmIkgiEUhAZ0gCMIcyMI9O7ZOBAnr9KBu/5eW1Jlf9aBXPMy5zqlU6iIY9n+6dXrvAhUPEbKVsi6BQED91Kc+ebqaQ/R8R0lJyZDf7z9azHOzMcVKHspKCGQaIw7gmNgAzea9A0aF+rli/wTmWr7PTzY6KUkj63pIqQoXSPndpVIpBqqmIAjCHuTyjKu/xvRqNIti27gpzTo9Y6up+cI4AP/8YySTyWrGWBiGw9Zo+iJiPW4S7lqWXGxLGKSL5zrBc4PTxTgXYtmY/ey80PFHxfYitM/GJhgx01YAV0JbTNMHI2Y6ibl/P2R9pjppcTOg/R12jICeSqXIwp0giEUhAZ0gCMIccn0As20ypABwQEuqWKWXIefcu5AF4AIVD/OJY27Fw6D+vS6Gc8498Xj8Yz6f7799Pl+v1AkWH2tcaJOxYCWL6Uics1Mq0HMhBuANsQFau4ZOzLUvBLRqjHTLdyclaWRXoMtInsjqgU4COkEQdiGX1lfpFehuxFIxo7BOL9VjpvltpyBaTaW5a83PPXIY1ulhxtikoigD6e5aiqKEZ2dnrwGQKisr+7nkKRYby1xrouCY8fy30DGdUIGeCxzAkNh2QxN4m2HETJcBuBparqYXRsyUSvt5M3FSbKaPJTP2Jwt3giCKCgnoBEEQ5kAW7tkx3YoxzTr9tACuf420xE5axUMm6/TovIqH4QUqHmZyqXhIpVI1AMCssmpADnYVO4nsOO0+NjshsJzkRhTA62IDtM+sTmjJoS4AF4r9MWgWhmdBSw7F8jvVRZF13WVVODiuBzoJ6ARB2IicBXTOucseoU9jesworNPLcnTXymSdnsRcdy3dOv20IJ7msBXNZfFlPB5P4My+5U6H4ibCTpgdN+VzfBWaffsggGehPXu3wLB8vwqaHjIrXn+2+Ho0z/GyQQJ6/pgeC5JrF0EQ2SABnSAIwhxyXSWpcs7dbOEOLPEBXFQ8BNN6453RQzw9yYMzky4q5lY8jCuKcmJ+xYME63Tp2b+XX365CgAuuOCCbD2UCYIwl0IkAmYAHBQbAFRASwpdDqASwIfE/iEYlRa9MJJFy8FJlof6OJA8lqnVFKqqUjUFQRB2IetnVdrzuJvjpiX97RULiX0ZBPGF3LUCGQ4TEzGTvnB4LH0RcXrcxBiLm7S4QVrMlBhK+LyN3jhTjCHVmKqoM6rHW+eVaQ3vylUiLqDQC4CKeZ/IGrsQz+YqgH6xPQ0tP9QGYCO0BchvA3AxgAgMu/ceAKcKMDZAArqlx0omkwpjjGImgiAWhAR0giAIc6AK9OycFtDTrNMX7I2HuRUP80kgreJBURTdOn2higdZc8zE6QoamYNGIhHl1ltvv5Ixxu+7738eCAaDxejz5NZ7nSgcZicHZFWgm8E0gAMAVgOoBvBrGBXqG6Alhji0aoweaMmhPmifn0tFprDtVAGdKtAJgiA0comHyMJdi5l06/RMleKnxfA0d61M1umRNHetKcbYQLq7ll41Lv4t9t8RLitmUmOqcvyPj7/Dv9o/0vEfHbuZwqDGVKX7k92XqVHVt+aeNY+mC+smQhXoRKGQ8Rxtx7gpBW1RcQyagH4XNFFdr1C/HtpirTCMmKkbwESe4zlRQJeVS5Jh4S5zPgRB2BAS0AmCIMwh1+rqXCwLbYeoePDP74eX/rWqqo0AMDMz8yUA/gyHiaZVic8wxkYzVTzkap1uIYqSFPnJT36yqr+fNTIG/tOf/rTz05/+9HGJwxd0zs8//3xNe3t7pLW1tRCVtERhKOQ1dtxnYhoykhv6351JAPvEBgC1MHoBbgZwCbS/QQMwKi1OQLNgzQUS0PNDioU72RESBGEjcomH9OS2454ROOeKHjMtsJi4DEBpMpnclkwmr0Rm6/T09lIjiqJ0p7eaShPGc7JOtxDSzlUJKGrN+2oOjt4xurX3z3rR/q/tL3Z/uvuy2Bux1hVfWvGEJPG8ICROJUpK6kvmxMfJiaTXU+FJMa+trr8TccrvX1YPdBkCfRLaouJu8b0PQDvmLkRmAEIwYqZuAEtx9TP7ukuJLwROs3An1y6CIBaFBHSCIAhzUJHbgx6HTawI06zTs1U8lAEox5nW6RzC/k/8O8M5r/N6vc8yxkIZKh6cugpUegVNJBJRfvWrxzYB13DOU2znzkc2fexjH+spQhX6suc8MDDgv/32H1zS1VU/+p3vfOMFRbHF28fJmHUfZ/v8tGsFugwWmsO42F4W39fDqLR4K4DLoCUP+mFUWgwgc0LBaRXospNOAFm4EwRB6OTyeag7ONniwU9VVd06PRd3rUzW6bNIc9finKcYY4Ner/fg/Epxxtisg/vCS3Vra7yl8U0AGPnxyNZDlx5ayzxMXfGlFU/U3lg7IOscBHnPOT4Y9/f+We8VVddWHdPnkxxLlvR9qW+Lr80Xavv7tv2FO03CBti5At1sFlpAGwdwTGyAVuzRAWMh8rli/wTmWr5PLzKOkyrQTRe105Bh4c4s4LZCEISFIQGdIAjCHDiMgH+xB76iWrhzzj3COn1OlcO8nuJ6kieY4VwTmFvxMKRXOaQJ4bqFeiy94mF2dnZDIpFo9/l8LyiKEpc4basg7brr1ee1tX8Y4jyF/v4nGiVXoRcs6Ln77rtXDQ5W1oZCE6XPPffc0W3bto0X6tiEZVjsfrF7hlhWciOX458S20vQzqsRRqXFFgBXwKjI6IGWHBqEUSnoJAFdiqg9byxT58U5B0hAJwjCHuRSgV5UC3exkDiwUKU4hBiufw2gZP4hoFmn6/3EQ4yxkwu4a0UYY3PcYMLh8F8oitLr9/v3yJmxpZB6zev/uL575McjWwFACSqxmj+okS2eL+v5oKS5ZLZ8S3nvxH0T6wGg9sbavr4v9W1JjafKaj5bc6Awp0hYjIXuGTPfO06oQNfJNsYsgCNiA4BSGIL6KgDnif1jMMT0HgAzYr/TBHTZPdBNjc+oAp0giGyQgE4QBGEOudoMFtTCXVin+xaxAJwjjCOzdXosvUqcMXYqrS/e/N54eQvfaWK63QWxpSJ1lXZ69bnP15YEAM6vKGYVet4MDAz4d+16bW15+Yej0eh+/44dv1q7detWqkIvIkKkcxKyrAJlWLgvBQ5gWGwviGM0w6i02AbgKmgVGX0wkgwyfl9k4Z4HogLdNp/vBEG4mhSKYOGeZp1etlDbKQiHLWii+PwHzhSEu5bebkpYp6cvINYXFEeWaZ1e1EXXRUTqg6be85x5mFq6sfRE9EC0o/fPerfoPdHtAFMYmv+y+TAATNw3sX7ivon1il9JtHyl5fmyC8qWYjktE3v8cpdPoedphd+bneOAfH9/UQCHxQZofx86YSxEvlDsH4UmqDOYr784VUCXYeFOPdAJglgUEtAJgiDMIb0CPdvrFlX/RMVD6RIqHuZ/tuvW6XrFw6SiKAPpFQ9pvfEiEu2L3C6gS5m3Vn2OxkBg3ezMzMt+AAgEzo6fOPF4UxF6oS9rznffffeq0dG66vb260bC4Y7Avn3/2k5V6JZB5ip0mePZkUJ8tnAAJ8X2HLS/UytgVFp0iX23wqiy6AYwgsJeGycK6LIs3KkHOkEQdiHXmAnIHjeVLMFdqzTDIWYx111rYn6VeJo4LtM63c0CupR56+K53vO89sbagZEfjqzWe6JLFtGXNRBTGBo+0fBm6PehtQAQOCswYmHx3G3IjGHMfI52wudRoZ7/ZwAcFBsAVMAQ09dC+7v1dgDnwIiZeqH9vSkUThXQTR9LCOgUMxEEsSAkoBMEQZiDLqBn/k9hnQ7Axzn3zc7Obp6f5MHciof5AUoScysehvUqhwwVD9FlVjyYhVsFdB0p83711YONgUA8zvm3WTxu2EkGAjx+4MCBRgAyBPRl33969Xkw+OGYx1PGKyvPi05MrKuweBW6G+5tp83RrRXo2VCh9UMfAPAMgK0ALocmrq8CcA0AD7SKjB4YyaHRZY7rZAFdRgU6JYMIgrADCy4oEu5aAc55BQAkEom1iUSCzV9EnOauNd86HdCs03V3rTBjbHieIB5WFEWPm5IZft4KuFVAByTNm/mYWtJQMlPznprTPc/1HuKJ4URQoni+7OcDvee54lcSvpW+icj+SOvID0em9PkQRAGxW0wz//gwYYxpAK+KDQBuA7BPjLceWqssfbGybvneB83pK19kxzKyKrZlCOgUMxEEsSgkoBMEQZiDmkgkmlOp1ArOeWmGiodA+osTicRKADHMrXgYy9AXT/8+LrHiwSxcKaDLXszwb//2ry8MDAzsz/R/ra2tMZnnshzuvvvuVcPD5bX19W2T09OvBQDA718f37fviJQq9B07drQdPPhG7Ve/+n8OWFSsdxK5fCZYcVHQUrBCD/TlogJ4SmxeAG0wLN+vg1ZtMQNDTO+B1htwKThRQJfyAUIV6ARB2AVVVQOJRGKDqqprMsRMc6zTk8nkdmh/f9IF8FOKovTOd9cSC4qjjDEnWLO6VUCX9rzHFIb2b7a/NH9/kUTnvK91ciLp1Xuet3yl5fngecGpoX8bOkvviW7WfKYem2pgJUytvKLy9LNe7FgsGH4uXF//8fq+RX7U7s/0bkXW55EVeqAXghMAXhFf18Bw9ToXwCXQ/q4NQouZusXrl7Kgy6kV6Ka3pKIKdIIgskECOkEQhDnwZDLZkkqlWtIqHkKMsZPpgngikdgKgAeDwZ9JtE63Cq4U0CF53sFgUF27dm1Exlg5kPecX3/9eENlZWwmHv/m6coirxeqx5NS9+7dW2emgD41NeXdsWPXhvFxVvHEE08MXnXVVafMGosAsLiDh9nvG6v22lvqGGYnNeaPkYRReQ5oFYDtMOwL3yl+ZhqGmN4NYHKJ45iFlL7kAilVG5xzqqYgCMIWqKrqSyQS50ATxfU4aUB8P6MoyoyqqmXJZHK73+//sdfrHXTAQuKl4mYB3W3zXtaziOJXVG+9N9L02aYDum178182HwYDPDWeQlpGzyH0u9DK2NFYA4A9lVdUjsWOxYIDXx14G/MyteaGmgFPuYeeSeTfyzKeo+0cN8lyBps/xoTYdEG9HkbMdAGAS6E9w/fDiJv6sfhzvezFwE6ycKeYiSCIRSEBnSAIwhzU0tLSPdAedhe0Ykomk+cD8LpQPAfEgzDn3G1JER03zXvZQc+3vvX158fGxjLZcqK9vd3USvodO3a0nzxZVptKtSg7dz645oorrjhFVehniXGqpwAAIABJREFUUOi+105FZqLGTLKNkQDwptgAwA9NUNcr1M8R+6dgVFr0AAhlGMcxfcnTxjJ9TmThThCEXfB6vaPl5eU7YPzNOINEIrEymUyCMZZwoXiuO1i59eHTfRd8GShBRW3/l/aX0/cxhWHFX604bOa4rbe17uv/P/0XDP/78IXxgfihqYem1jAvU9v+sW03iedzMOMZsBhV/E5YFCwzZlpsHqfEtke8vgFGzLQFwBXQFiufgBE3DWJu3OLUCnTqgU4QRNEhAZ0gCMIccq0yVl0sILu9At01pCU6877WVVVVyaqqKul9KaemprwPP7xnrdd7faK2dlP00KF/bXviiSfqqQr9NMWopADMex/JSgjYuV9gPmPMAjgqNkBrY9IBw75ws9g/jrmW77J67DmuakM8W1AyiCAIO6Aih5hJ/Ou2uEHHjZXYgAvjJoHtrrUSVNS229te7vlsz7bxHeObAKDzB527fK0+06reCTAsfq/YvQLd7OMXqwJ9MTiAEbG9IH6+GUaF+jYAV0FbrNwLI26S7ablJAt3BUuzyycIwmWQgE4QBGEOugVxtuDXzdUEbhfQ3TZvW6JXn7e0vGPM661KjY2tr6QqdGIZyKp0kCE6LydxEgPwhtgAIAgtMdQJLTl0vtgfh9ZHfT20BJFZ7SgcV7VBFegEQdiIXGMmwN1xkxtjBzfOm3Nuz3UD8cF4QI2qPv372NFYOQnojsUKjleFOr6VF09zACfF9jy0v4ErYFSoXw7gGmiCOqD1VH8dwPAyxswGVaATBOEqSEAnCIIwBxW5Pei5MSmg43Yh2a3ztjwjIyO+u+66q+uGG27o1avPS0pqUgBQV/fOEFWhWwK7JgRkJWqsVoGejQiAQ2IDgHJoSaFrAZQBuFHsH4ZRadELTYgvBLIFdBk90CkZRBCEXcj6WSUszAH3Pj+7OWZ067wtixpTlemnpuuq3l41qu+LHokG+77Qd5W3wTvd/k/tz458d2Tj8L8PXwjRE72Ip2slzIgzir3awu6uWjB5jELHGCqAAbE9A8ADoAXa4uPNAC4BcBmAKLRYSbd8H810sDyRtVhbH4t6oBMEUVTcunqXIAjCbHKtQM/FstCRuDgRVuwgt5hY6loPDAz4b7vttnMmJibmLCi855572h96qH/9P//zP58/OOirjcW6/T0932rs6flW49jYo5WTk+Vl9933UFexztuiLHpfP/jgg01f/eo/bFLVnGPthY5nqXvIojihJ2EYwGvQ+gEeA/AtAP8LrfriLAAfBnArgE9BE9nXAvBlPFJuOK4CXQjospJbBEEQyyFnC3eXt75y49zdGjdZ4lqrMVWZemSqcf7+k18/uWHkxyObT/3s1EoA4CrHiS+euDJ+It5Y866ao4GuQLTt9raX/Z3+8ZEfjmxOhVMe+WdvWTLe0+Hd4erQE6G69H3xgbh/7OdjbVmOVywLd0vco8vEKj3Ql0MKWm/018T33wdwF4AXoTl8XQfgMwD+CsAHAFwIoO7MwywJmRXogMnxjKqqFDMRBLEoVIFOEARhDrmKw2Th7ozgaylwwJUJQMslwL7//e+ffd99+y8IBn+UuPXWW18HgKGhId+uXYdWh8OrAt3d++uvuqr+9dLSmSTnYXR3d9e0t7dPer3VfOXKldOLHNpyczWRrPdxJBJR7r77txsGB1F9ySW7hq699tpCroAvNLIq0M3Eij3QlztOCMB+sQFANYz+6ZsAbBWvG4RRaXEChp1hNmT+HZbSA50s3AmCsBG5COhk4e6+mAmMMU4xU/EY/vfhdWM7xi5MDCeerP94fR8ADPzDwMaJ+ybOL99WfnjyvsmzACA1lfLBg1TdH9U9V3tj7SBg9EQP7wlXeco99DyShcnfTHbEDscaAbxSeUXlWHwg7u//Sv9FSMJTtb1qyFvlXaxHczHvGTtXoFuxB/pyx0lCc+vqAfAEgBIAK2FYvm8Xr52G4erVA2BiCWNJiWUgqa97KpVinHPqgU4QxIKQgE4QBGEOZOGeHbcKyTpunbcl6OnpKX3ssb0bpqfbyx544JlNN99887G6urrEzp0720+dqqtcvfqzQydO/FN9S0tz+HOf+9yxn//852133PGzq2+4oer5b3/72y8W+/ztxK9//evWvr6S+kSi3fvrXz+8+uqrrx5dRv94mZXCZuAUcVumgJ6pImASwF6xAVolRSe05NB5ALaJn+uHkRzqh5ZUWso4ZiDLwp0EdIIg7EIun4luXXir40YhWceN87bEnJu+0HQ4ejjaMPSdocsBPDnbO1s58auJC6rfVb2v9Wut+4e/O7xWF9H9Xf6xke+OvCPQFZiuvbF2AAAi+yNV3X/c/dGGmxseb/lyy8GiTsbitPxty6v9t/WfN/K9kfOTo8lDkw9NdiEJT8vftbyYRTxfDLtXoDtJQDcbfZz5f08TAI6LDdAcuzpgxE2bxP4pGGJ6N7TFy4uNJSsOhNljkYU7QRDZcOvqXYIgCLMhC/fsuDIRlmZd70Ysc61//OMfr5uYqKtYseLvR0dGSmvuuOOONXr1eTB45azPV5sKBq+M7dp1aPXg4KDvhz/8zy3hcEX5gw8+fuHQ0NByrKJdRSQSUX7zm2fXKspWtanpDyYOHYq27tq1qyHLjxXzPpFtSWcWThLQcxlnDMDLAHYC+AY0+8JHAMwAeCuAmwD8jfj3MgDt0HoGLnWcQiDLwp2SQQRB2IWcLdxzeJ1Tceuiayc8k9kWxafwzh92Ph08N9g79K2hK3XxvO0f2vYDAPMYcW3pWaVj3kbvWO/ne/9w/JfjreHnwzVvfuTNP2IlLFn9nure4s3CUiz4HlaCitr2tba93jrvzNjdY5tSE6mylr9reTGwJhDJ53gSsPui5nTsLtIvZZw4gKMAHgPwIwD/DOAXAA4DWAHgfQD+AsDnAbwbwEYA5RnGkimgy7Bwp5iJIIgFoQp0giAIc8gpySNs6dy6mMmVAjpo3kVHrz73+/807vN1Jj2e7eoDD9y7yePxqKdO1VV2dFwxCgCNjVeGenoeb7z99tvPP3LkZEdp6b9Epqe/Wv71r399M1Whn0HG66tXnzc3Xz/p9zcmT53aVJlDFXqu7h12xCnV4VYT0OczKrYXxTGaYFRabAVwJbSKjD5olRaleY6TD2ThThAEMZcUyMI9G24W0N02b0s94yo+hfvb/ZORvZqOG1gbmOAqx8gPR1ZPPzm9qmp71ZFUKOUL7QqtbvxU47MjPxq5pOeWnpsAwFvvHV9z75r/CW4Mhos6CWux4PVNTiRL1Khaon8f74+XZhHQnY5TWmsB1hHQ5xMD8IbYAK1vegcMy/fzxf5TMKrTZTl3ybJwV8jCnSCIxSABnSAIwhxyrUB3Y1JAx61Cso5b51109Orzxsb3jwNATc0HQ8PDD9b+8pePng9cxwYH76/RXxuPV3kfeuixC1OpczylpR+IpVJHPA8++JML/+ZvhvY1NzfHizcLy7DgfWxUn1+u+v2NSQCor78+dOjQv7Tu2rWrYZFe6Iu9N8x+38hI1MhKoJg9hpUF9HQ4gCGx7RbHXAFDUL8MWn9AAPgIjOTQUAHGzoQsC3eqpiAIwi5krUBPc3By6/Ozm2NGN87bMnMWPc/Pq7q+6kBiKFEx9J2hyyMHIwfj3fG6qu1VRxr+uKGXq9rbM7QrtLruo3UvnPzHkysBoOFTDY+TeG7AOQdjmS/t6Z7nHKztH9ueGf2v0XUj3xs5H6Inep5DyrBwt3Pc5MQe6MsdJwLgdbEBWvV5p9i6AFwo9l8KrYVWj9hiyxw3E7Is3ClmIghiUUhAJwiCMAddQM8GWbi7b/5unbcl+t339vYGHnts74ZUapN3aurXp+3I4vG2ksnJ3RWXX97/WknJyGlxa2Cgv+r48VMrA4H/O8uYgtLST8Wmp++mKvQceOCBB1p6ez0NicSQt7v7e436/kikwn/ffY92LSKgA8WtvrFU5U8eOEWk18cp9Fw4gEGxPQetuuHt0BJCHmjV6ddCSwT1QhPTu6FVtBfiXMjCnSAIYi65xkyWeJYsEm4V0N04b8s8hw7/x/DadNt2Na6ynlt6EHostLH2xtqXGv64oRcAmMLQ9Lmmo1DBhr49dJ1SoYRZCUsMfWPo3f52f1jviU4szNB3hjbqPc8DawKRtq+17e2/rf+8kR+ObA6eF3xyGX3QzcYy92sekICenTCA18QGAJUAvgBgEsA6AG8T+0/CWITcB2C2AGPLsnBXOOcUMxEEsSAkoBMEQZhDrn36ONxtRQhQUsQtWGLek5OTJbW1paFg8EgMOHJ6f309psrK2ma//OU/39vZ2RkFAFVVceWVb/8AY2czj6dDTSZf9QAA55exBx/8HVWhZ6GtrW3m2mvrXgcmoG06pVixYkW+1ShmJyHsXuGu43YL96WgAggBSAL4KTQRvQ1Ghfq1Yl8ERpVFNzQrw3yQYuFOAjpBEDYiCbJwz4YbhWTAIvGDW6m5oaaPpzhr/kLzEcDoiT70b0Onmv+8+Y301868MFMz/J3h6xS/klhz75r/KWksib9x/Rsf6v187x8C+DmJ6AAWeQ83/1nzQXVWVXTLdr0neuRgpCIH8Xyh94mMCnQzcULcZHcBfT4hMdY+AK8AqIYRM20EcDGMxco9MAT1RB5jybJwZ9CeQwiCIDJCAjpBEIQ5kIV7FlxsxejWeQMWmPO55547/fDD9z6Qy2v7+vpKe3r6WrzekmQ8/h6fvt/jQWp2NuV78MEHW2+++eZuM84zHo+z++67r+W9733voM/ns0Py8IxzvOSSSyYuueSSiUwvtjhm/76dIm47RUDXx9EXvqWgVZ73AngSWry0EkYvwHdAS+iEYSSGegCM5zGWaQgBXUZ/QoIgiOWSa8yEHF7nVNwaM7p13paYs6/VN6uL5zqKT+Etf9tyaP5reYozb4N3fNWdq+7TbdvXPbRux9Ebjr6fJ7M6R+Q138RQwlfSXBLPts9iZHyu9a30nWGBrQQVtfyt5VOLHMsK94md4yaZvz+nCOjzx5mEJqbvE9/XwoiZNgO4BFo8MgAjZjqB3ARrKXOiCnSCILJBAjpBEIQ5qMjRjtDlVoSANQK/YuC2edtBBJ5DZ2dn9L777vnJ2NiYb/7/lZSUqNu2bctVMFsyP/rRj1b/+Mf3bhseHn7mc5/73DGzxikABb+POc/pVrFrNYWsag0ZYzhNQF9onCQMG3cA8MEQ1FcB2CB+PgRDUO8GsFDCkyzcCYIg5pL1s4oxlqu7l1Nxq5AMuG/etouZAKBiW8X4hhc3/Cx9X0ljSXz9M+t3mDHeyP8b6Rq9Y3Rr+7faf1uxrWIcACYfnGzqv63/upa/adlFFe+m44SYhizcCz/OuNheFt83wKhQfyuAy6D9zT8BI24aQObnAFkCOvVAJwhiUUhAJwiCMIelVKC72YrQjb0MbZkUcSubN28OyR4zEokoO3c+dO7oaF3tzp0Pn/uJT3zieDAYdE0lKWOLfiTI+LwwO4nihAp0KwjbxRonDuBNsQGAH0AHjOTQOWL/JIxKi24A02I/WbgTBEHMJZfPRLJwd5+QDFDcRCxA9burB8Z3jof7vtC3vf3b7b9NTaZK+r/S/w5fm2+08prK4WKfn0WQ8Rxt57iJBPT8x8k1NzEqtpfEzzbCqFC/GMAV0BYr98GImwbF8fW/92b3QGeqqpKFO0EQC0ICOkEQhDnkWl3t1mQI4NIK9DRx0FXzFrhtznnN98477+zq7+eN9fW3TvT339Z45513dlm8Cr2gFHlRjd0FeoAEdNnjzAI4IjYAKIUmqOvJofPE/jFoSaEqSLjPSEAnCMJGLEVAd9uzpI5bY0a3ztuNc14SvhbfbNddXQ8dv+n49T2f7rkBAPxd/sGu/+561FuTtWd4sXDSdZU1F7uL204V0PMZhwMYFttucaxmGK5elwK4Gtpi5T4AJ5d7srmQSqUoZiIIYlHcunqXIAjCbHK2cIezAqml4PZEmNvmXdQKkng8zj7+8T+9/I477lhVzPPIhl59DlyDsrLLY8A12Lnz4XMjkYjVn9kKeX11B49MmJ0ckFXpYDYkoC+NQlaFRwEcBvAQgB8A+AaAndCqKjoBrAFQD+AzAK4HcDY00b3QUDKIIAi7kEvc5GoLd8aYW4XkYi+sLAZFiZl4krP+v+s/d7Z3NpC+f/g/hteGfh+qN3PofH/Q1+Kbrf9Y/Sv6902fa9pjYfFcR+b1tXsFuiycEPvJEtD1nEQhxuHQRPLnAPwMwD8D+E8AT4txLhavuxHAhwFsAdCEAv9OqQKdIIhsWD0ZSxAEYVfIwj07bhbQnRBo2opf/OIXK/fuDa3++c8f2jwxMWFZBx69+ryq6o/CAFBV9Ufh/n618c477+wq9rktgBvfv8tBVnW42ThNQDdznBkABwH8BsB3AeyHZu/eD2AtgA8C+GsAtwC4DsBboNnCLwuqQCcIwkbkGjPpn21uxK0CuutiJuFWJv1az/bNBsLPh1f1frb3al1EH/r20FvGfjZ24fTT0ytkn08uTD442TT0raErS5pLxrz13sn+/6//7dPPTNcW+7yKQDHeJ07qT24mdqgMt8o4KrT46BkAPwXw/8T+Q9Bio2ugxUtfghY/vRVaj/VlQRXoBEFkw7IJZIIgCJuTkzjs5moCkIDuxnkXZc7xeJzt3PnQxkTiXO/Q0Ot1d911V9cXvvCFI9l/Ui569XkyucGbSo0r0ei4DwCSyQ1el/VCL7aFu92TtWThnt84st5bKQARAPeL76tg9E9fD626Qq/I6IFWud4Hzc4wZ0hAJwjCRizmPAPgdMwEuPP5GXBv7ODGeRflOTTQFYi2f7N9V98X+67u/Wzv1cELgidCj4Q2VFxVcbjlKy2vFuOcFmPywckmved51393PapGVc/xm45fr/dEr9hWMV7sc5SAFd4bdo6byMLduuOkj7EPQC80DWsljLjpHdCKkWZg9E/vBrCk976qqgwUMxEEsQgkoBMEQZgDWbhnwe2JMLIjlEM4HFZ+9atftXV3J5sbGj4xMTX1u/IHHrhvw0033XS8pqbGUlZdr732WuXUVLgsGNwXi8f/zKfvDwYRm5qKlx84cKByy5Ytk8U8R0ksdq/IDNrNwCnitqzPr0Jaqy+GzIUT88X6KWhV6fvF9zUw+qefA2CrOLcBGMmhEwASCw2gqqo+DiWDCIKwA7ksYHK1hTvc7Vrm1msuneCm4HT7N9t39Xyy512hR0IbgucHe9pub9vLFOtdAk+1J1F6dumJju91POWt8SZRg2TXXV0P9f1F36WeCs+Cz0hFRvYv0sznWydVoNt9jPRxzEZmLD5/rCS0WKgbwOMASgC0w4ibNoifmU57XQ80568FSaVSAMVMBEEsAgnoBEEQ5kAW7tlxs4BuV/FvuUi91vv27at417v+4E8rK8vDjP0JDwTWJDyeytDQ0COWrEK/6KKLJu+9979+GYlEPPP/LxgMprq6uqLFOK9cSFsQU5DDFfBY+YxtdhJF5op9s3BiBbpMAX2xsSbEpvf0rIdRaXEBgEuhJXn6YVRa9CMt8ZNMJvX3ECWDCIKwA7l8/uqvcXPcRDGTe5ByraOvR8uOfejYBzv+veP+yisqxwBg8pHJFbO9sy2eKk8o3hevjZ+IB/wd/piM81kKFdsqxiu2Vfw+fZ+vxTe7Zsea3xXrnHLEafe02QK93eMmmQK6TFFbhnNXtrESAN4UG6DZvOuC+ipoC5EBTUDvgSGoh9IPkkqlaNExQRCLQgI6QRCEOeRaJeHWZAhAArob5y2VL3/5y5dPTzdXhMMny88++/ojAFBS0pjyeK5LWbUK3coi+QKYdR9nSwA4LflUSJxS5e7EcYClV9WfEtse8X0jDEH9bQAuh1aRcQJAz5NPPjm1YcOGw+K1bmj5QBCE/cnls8rNcQMAcBe6VwHujJmkPeNGX49WJkeS9W9+9M2bVv9s9V3hPeG6oX8dei+Pc3/5JeUHUlOpst7P9l7d8b2OXVYU0Ymi4oT3pdN6oBejKtxM9AVzuY41C+Co2ACgFEAHjLhps9g/DqD7yJEjw4qi9AgBnWImgiAWhAR0giAIc8i1At21Fu4QD8IuTQYB7rvuUgXPffv2VezZ070e+ABT1ceU3t5PtVVXXxYCAFWNKiMjnpodO3Z03HLLLW9mO1YekLhrHjLsAu1egS7rs8VJwrYsq3hg+f3WR8T2ojhWE4R14eTk5CUf+chHfKWlpWhra0N/f/8XGWO7AOzlnFtqsRBBEEQaWT9/05xuqALdfbh13qZTe0PtSf7/s3feYVJUadu/T3Xunp6cEwNDkjQIqLig6ErQNaEbFNY1p/3WNayv+67uvrruugYMuCii7opiAhVBRZIgDkEJCggqaZjADJN7cufuqvP90V1MM/RM98x01Ux3nd911dVMd3WdOp2o57nP/Txe+l7lvZW/LZ1XejP1Ug31UnXi5Yk7Ct8tLLb/YDdX/qnykroX68YPWTjk24E+X0ZQQrW+kvr6Npod6LHicgfkjZkg01j9fX8cAI74NwAwolNML/jPf/4z+b333sPQoUOpSqUihJCjAIoppU39OWkGgxF7MAGdwWAwpEEU0MPZT8mJIECZSRGlCqyyvdePPPLIDLc7S6tW/9nt9eo1Hs9S7aWX2vbr9XoB0AIYjqKioma5zofBCCAWEiix5gwfTCXcewMFUOffdhqNRtXSpUvH79y5c9yHH35YCOD/ADwFoJ0Qso5SOi9C4zIYDEYkCfc3UckisiLnHuEWQdGEbO91ym9SaiBgecX/q7gRADGdZzpY+G5hMeDriT7k5SGbtNla5j6PDLH0HY6FBbuxJNLLtRh4IHqgR8odbgdwyL/hzjvvTB8xYsTkDRs2jC0vLx8BYCUAEEIOAriHUro9QuMyGIwoR6miDYPBYEgNK+EeGqUL6Eqbt2wJMNF9TsivqUaTxet0d7o8ngx1bW1twpNPPrlf3M4///yW/o5VU1Ojmz//dxfv2LEjORLnPtAUFxenFBcXp/ThqXL1QJc6aJfDgS4HTEDv/Thyle6TLMGl1Wr5OXPmfH/nnXd+1NraKgBIAjAdwLPw9fzrN4SQ6YSQtYSQZkKInRByjBDy10gcm8FgKJZwf3+VWsYcUGbsAChz3rIuGhCcAqldUPtzcWzHD47h7cXtp2IBwyiDTWVW9bk/sLfNq67+R/V4wS2ceh+pQFHzVM0Yd7Vb149THxDc9W6N4D39J0vwCvA2ecM1qEXs/Q3j9zAWHOhyEAsCOivh3ktGjBjRcOedd67nef4nAP+Cr3/6TQD2A2iMxBgsbmIwYgMmoDMYDIY0sBLuoVGygA4od96SI7rPNZqb3QDAcUMpcCWKi/dPrK+v10RyrNdff33Ud9/ZRr766tKiSB53ILBardytt9573S23/HGe1WoN6xpRokR2LLuNYsUdHosCerSUcA+Jx+PhCCECpdRNKf2aUvoEpfTh/h6XEDIfQDGAFgC/A3AZgKfB/j9jMBj9I9zfXwHKzWEpUUgWUeK8ZZmz4BTIoZ8d+p3rhCs38fLEbwpeLXgLPFSlvy29KVBE7w8d2zpSWz9rLaq4u+ICwS0QKlBUPVQ1pXll85S29W3ZkRhDLrytXtXh6YdvP3bZsatFEV3wCjhywZHrD194+KauwnqMI0dbLSD6FzYzAX2QjyUIAgHAU0qrKKVvU0pvppQeCfnEELC4icGIHZQafDAYDIbUCAi/n58if4sDSvIp8QJSqUkwyefs9Xqxb9+xsygFcbvn6ZzOmTqnc6YO2A6n02B49NFHz4nUWDU1NboNG74bKwjjVAcPNhZs2rQpLVLHHgj++te/TmltNSe2tSXE/+Uvfzl3AE8l6Ofk4MGD8e+8806O/89odaBDhuPLlQyKJWE7Wku4B8Xr9UZcpCeEZAN4DcASSukNlNK1lNKtlNKllNInIjkWg8FQHGE70KHM62dAuXNX6rxlwfKWpcBd6c5J/EXiN4XvFBan/CalJn9R/nvgoar5V830SIyRdGVSXcb9GdscBx35FXdXXFD5p8pz24vbx6TMT9mddntaeSTGkAt1opo3jjeW2Q/YR4si+pELjlzvLHUOMZ1rOsapBzat4230al2HXXGB91Gewr7TLmWltGiOm2JFpBfHicay6uGMJem8eJ4nAPpcZSMYLG5iMGIL1gOdwWAwpCOcgF/JSQGlC+hKQ5Y5q9VqzJkzbY/FYokDPAGPEHBcLv3Vr35VEqmxXn/99VEtLYnxWVn/Z6mvfyR52bLl42bNmvVVpI4vBRUVFYYnnvjX5EceeXjvsGHDHOL9VquVW7PmqymUziUAwdq1q6Y8/bR1T1xc3EBZKc74vCxY8OJ55eUto8455xyMHj16IM4pEsSKuB1rwracArrkPQr9DvSIJoIA3AHACGBBhI/LYDAYrAd6aJQ6dxYzSUj63enl+jH6V+MvjD/V1irlNyk12hztf03nmfrd6kok9YbUSgDb6hbWXQQAKden7Ml6MOtopI4fSfgOXtX0ftOQ9LvSy8T7qEDRsKRheNodaaUjVo3YWHJtCTp2dEz8Pvv70QCQeHniN8PeHBZOv2RJv8P1D9dPc5e5s7IWZa3Rj9N3UIGSxicbs63rrFfnrshdoR2idYQ+StjI5UCPlTFiKTaDTGOJK1IkzUeIDvQIH5bFTQxGDKFI1yODwWBIjdls7k0Jd1CqxNyAogV0QLnzlpxly5ZtXRuENWvWrJs1a1ZE+lmJ7nOt9iqPSpUgxMfPswZzoX/wwQe5t9565wVOp3NQXHP9+9//HrdlS1nRokWLxgfeL7rPtdpbXFrtza4+uNAl/RHbtGlT+qFDlmFtbemGjz5aLeVQcvRAj4UESqwJ25KL2gFIXsJdCgc6gAsANAM4ixCynxDiIYTUE0KWEELMER6LwWAoC1bCPTRKFdABZc5btjkHiuci5mnmlki6qan4WwaGAAAgAElEQVRAYd9vTxf/dhx2pIs90R0HHXrbNttpLbacZU5DzdM1Y6ggf46k4fWG4XUv1M0++djJCYDv3CvvrzyvflH9zOblzfkAUPhh4UZxf6Il7oL/FIQjnotEelKnjpf2cNpuoiJ87b21VzoPOOObXmgy2rbYkhJvTNwRYfE86PgRRi53uFyVwWIlNpO0L3kXZC3hHuHDsriJwYghlBp8MBgMhhyEcxEmXgwq8fdYyQK6UpNgMTNn0X2emHiNFQBMpp85nc7RumXLlo8T93G5XOTddz+dsG+frXD58uX5A3e2Po4fP27cvHnfeJdrmG7z5v3jSkpKjECg+/wSotGM5zWa8Tyll5C1a7dOCaMXuizv6bJly4vc7rHa5OR77du3H0FxcbGUpQilRo6EQyyUiRfHiSWhXpaxeJ4nhJBIC+g5AEwAPgSwHMAl8LkqbgSwNsJjMRgMZcFKuIdGqXNX4rxjamW92PNcLNue+UBmsVjOXXALxL7bHtfyVoup8c3GAsAnnlc+UHmR9WtrgafWo5P7fDMfyDyaMCdhf/OK5p+dfOzkhMr7K89r29h2dvL1yd+k3pR6QizbDgAqk8pG3VQb2BNdZk77bmgLtfbsV7PXEBXhq2+rvs76hVWbcm9KVfLvkyNWga27sSUkmkvEA7EnoMdcD3R/CXcWNzEYjG5RomDDYDAYchFuCXeEsV8swuauLGJmzk1NTZqNG78d43IZtfX1zybV1Pwtpbb20RSv163+4YeGgh07diQDwPr161OrqtTpXm+RatWqzWMG2oW+ePHise3tqXFpaQstHR2pcYsXLx4HdLrP1eoLvF7vMZXXe0ylVs/wDkQvdEopoZTqEPC7ILrPTaYbbSbTdLfNNgzvvLN8rESnIIcDXWpipUy83OPIlfmUpYQ7Iu+k4ADoAPyLUrqAUrqNUvo8gIcBTCOEXBzh8RgMhnJgJdxDo9S5x0z8oFSaVzbniOJ51oNZR1NvSK0Ue6I3vNIwIvm2ZItpusltecsypfaZ2rMqH6i8CAJI/gv5xdocrUvu8yUcQf6L+btFEV0Uz3Mfzz0Y2PM88fLEb4rKi142Tzd/H9gTXU4opRp0ac2qGaJxqNJUbeLf+kl6u9SnIdFxYylmApiA3heiuYQ7i5sYjBiC9UBnMBgM6eARZgn3MPaLRSjgE8wG+kQGAKUmwWJizlqtVrjggrElNputAqg+7TG1ulBIT093ejwefPLJl9mUXkgzMuY2V1Y+kL58+fL8W265pWIgzll0n2u1/8+tVqcLWu117s2bF48rKSn5cefOXcMJARWE/+MEoXNxJSGU7tq1uxDArv6OTylVCYIQJwiCmVJ66pZSagYQRyk1U0rj4FupTeBrYH8cwPH331+Z7XaP1cbHT+sghJhSUq7HgQN/zt+8eXP6zJkzG/p7bjITK+XVY1FAjxkHutfrlcKB3gRgOIAvuty/HsCLACYB+CrCYzIYDGUQ7u+VYku4E0Ioi5kURczMOflXydXqJPX6hFkJp9popd6QWqnN1m4wX2hucnqchYm3JDo8Bzx1rWtbxwHA0DeGrtWPkFz47TWCU+AEl6AN7Hku9kT31HkSwzhEyPeVUgpKqUEQBLMgCKdipK63AMwAxNL34wAcpzw9Xntf7RB3iTvPfJX5W3ep+7K6B+qGpf8j3awfp+/o88T7OJcIHZ+VcA9/nFgT0GUZy/9/a6QFdBY3MRgxBBPQGQwGQzrCSQaJIjJHSMzEyWFBCFGyAx1Q3rxjxkFiNpv5BQsW7O3ucUqpauPGjSgvF+JTU6+t0elyvIRcgFWrNo+ZN29epV6vD/nbUFZWZnjqqacnPfzwX/YNGzas3z3rRPd5SspvWnxz+LWtqemDpMWLF49btuytT4uLizODPe+iiy6q6+m4lFI1APA8n8nzfGrXBA/84jgAQ5enCgCshBArIaSD47hqQkgHIcTq9Xoz/cdNOXz48BXV1U6SkFDIU/qZzmYDbzBwsNlSjO+8s3zCzJkzN/f3temCnEKqVMg1h1gStuUW0KXugS6FA/0AgPN6eDzavzcMBmPgYA700Ch57kqbd0z9f0o4gkDxXCT+5/EWAIAH4Jt54mn0nOoL3LGjI10/Ql/R03G9bV518/LmIel3p5eK91GBouGVhuFpt6eVcXquT9daYs9z0XnOt/CG5hXNPwOA3MdzD47ZM+btrv3hR6wasVHwCuipb7y/ypYKgN7tdo/oQRiPA6Dq8nQnIcQKoIMQ0sZx3ElCiJVSquF5PgW+78iklv+0TPeUeWj6o+knzVeaOW+j11F7Ty2tvbf2ypw3c1ZL1Ac9mj+vcl7/x5qALke5BTlLuLO4icFgdAsT0BkMBkM6WAn3nmFzZwwYL7/88vCmpibDY4899kOkj22327nPP/8KhEyDTpfjBYCUlGvbKiu3h+1Cf/nlxWOLi6snmM2LXS+88Px3/TmfiooKw+bN+8Z7vaPUra2L48X7vd4U9ebN+8fde6/uh9///vedySef80Hvd4mnu1yunpwPWgDweDxzxcPCJ4x3+MVxiyiSE0KsHMd1cBzXQQhxBCyiOQ1BECbxPN8MoGL//v0jKbVeEh+/xajT7TBwHKcCgMJCt0alissHkAyguT+vj8zEirgda8I2c6CH5mMAdwG4DL6kkMgV8M1nd4THYzAYyiFsAV2hLmxAuQI6i5kGiJbPWjL1w/VWwxiDVbzPWeI0On50JCRdk1QbqXG8DV6VZYElDgJIwesF6xqXNo62vGWZAgBpt6RVdPe8prebhja83nCBu9Ztyn0896AofLd/1T5WnaZ2plyXcrIv51O3sG5UYNl2/3HRvKL5Z/ph+rbUm1JPBO5/qsoWFcxetzdklS1KKdxu90j/020Bwngjx3FlYrwUEDdZCSHeYOfq8XhyeJ53wXcNRnSjdePUGepC85XmZABXqNPUJGtxlqP9o3ZOk6fJAlAJX6wWCWLBgS7H8ZkDvf9jSV3CXYqFxyxuYjBiCCagMxgMhnQICBFYBCS4lZwQUerclTjvQTHnmpoa3cKFr892udy6q6+++sTEiRPbg+23d+/ehO+++y7lrrvuKuvN8VevXp1TWmqD212uOnnyr6ni/U6nUfvZZ1+OEgX0hoYGbWJioker1Z4WgJaUlBi/+urgWJfrLM1XXx0cW1JScmjEiBF9LmPo9XpJYWFGTXZ2o0WlKlar1WqNSqXSqFQqjdk8ihBCZtntdn0XYbyr88EVkODp4DiuJsD5cJFWq12uVqsrCSHOSFbTmD9//rH58+cfC7jrPACXAmgAkAvgjwBa4C/3DqAcvvLvfUWOHuixkECRq2e4XMkgyfuSyzmWFA50SulmQsjnAB4lhKjga+1wDoBHAayhlH4dyfEYDIaiYCXcQ6PU2EGp8x7QOVMvJVV/rroaAsiIT0a8axhjsDpLnMZjVx77HeUpl3BZwmucnhPc1W4db+XVhlEGm/hcwS0Q+157Ytz5cS3hjNXyTksqFSjyX8gv1o/Q2/Oezttb9ZcqNL3bdHb8zPhaXZ7utD7oVKAgHEHGHzNK3DXuuJZVLVMAgG/hDe1ftY9N/V3qzr6K5wCQfmf6cVWcypN6R+pJr9ebTCk1ZzyTYdWdpztpvMo4wm63TxLjJf9t2FW2PB7PeYSQFr1e/znHcbYILXYUrymp6SLTDwDExeF6AH9Qp6ptyb9PTgfwO/jE83J0xk2RWIQsdQ90qeMyqYk1AV38PzhmxpKihDuLmxiM2IIJ6AwGgyEdIQV0dF4MKjEZpGQBHVDevAeNg2TBggVFHR1qM6UJeOaZBVOWL39/S9d9BEHAiy++evbx4+2Z48aNa5k2bVpYSSAAOOuss9p+/Ws3mpuby9va2io7H0lHWlqaHQCsVqtq7tz514waNaT2zTff2Bb4/CVLXh3T3p4Rl5n5jKWp6Y6kJUteHdOTC51SygX2F6eUnuqZByAuKyvL/O67S8WSgF0/d3ZCiI5S2kEIaeI4rqIb50NQUdrj8eTyPH8Rx3GtHMc5w32NQtDTd6PNf/sBfEmgAvj6i42ALyjl4XNXlMCXGDqjZKQCkCsZFEvO8Fgr4S6FAx0Afg3gMQB3wJcAqgHwPIB/SDAWg8FQDuGKpEoVUwFlz11p8x7wmImoCR3636Eflf629Hclc0tuyP93/qrKByqvEWyCYdg7w94Ry6PXPFEzyX3SnZT3bN5Ww2iDTXALpPL+yqnO4870wncL12vSNe5QY6XcnVLvtXqN+gJfz3PCEeQ9nbe3Y1tHecWdFbNTb07dLwriVKAov738Ak2mxpr3ZN7+vKfy9oOCiCJ66u9Sd2b9Oetwd2MF9Bc/LW467ZZQs/G3xji73a4NfG7c1XFeAUIHoacqa1kCKm51cBxnDVVly+v1FhFCHCqVKlL9yAm6/3444VtUfBzAZgDp8MVMwwHMgc8ZG7gIuQJAyPery9jRDuuB3rdxEEtjSVTCHWBxE4MRMzABncFgMKQj7B7oiI0ApLcoeu6UDnhuRJHU1NTo1qzZMoWQm6hKlSls3/7Pou+///67ri70devWZRw+bM+127MM77//4ahp06btCneMSZMmtY8aNQoajaZSp9MdCLbPq6++OqK6ms+wWI4k7tmz5+C5557bCnS6z3W6P7i02hRqNP6a37HjtcllZWV8dna2tmsJdf+tscvhKQKcD4SQ2gAhPFAYtxFCIhUsRvIDHc6xKHxJnmP+DfCVcxcTQz8HMBtAO053p4cS+ZkDffCMIY4jl9NdjnHEsaLOgQ4AlFIXgEf8G4PBYEQKivD/71di3AAod+5KDZgG/L02Tzc3F75X+E7J3JK7SueX3gkAI1aPWBJ/UXyTuE/WX7L2Vz5QOaPqoaoZuU/mbm9Y0jDOecSZlXFvxs5wxHMA4EwcVRvUp73PhCMwTTa1gQDVj1ZfDmBt8q+TT5bfXn5hx9aO8Wm3p30F+AR1wS5ofAeCiuf5VLfbPdwfK50hkCN4f3EXIaQDvtip3V9lq6PrgmJCiCtCVbYG6jPd4N++ga8FVwGCL0IW46aGEMeTWtyUy4Eul4AuNXLGTEAMlXCnlLK4icFg9AgT0BkMBkM6wnGgC8CpskFKQ9ECOpQ57wGfs+g+NxjuchBipnb768auLnRBEPDBB5+e5XYXqZOTL2nbu/fZoV9//fXR3rjQu4NSCqvValy9etNUtXqOxuv9Ubd8+QdXjB8/voFSav7sszVZlBYkjhp1JVGrPVx6+hWoqFiHdevWX3XzzTfx8JVQF4XwE904H+zdOR+ihJ4+Jz091gxgj39TAxiCTkF9Enzfuyp0JobqMDBJLNYDXbnjADKUcBcEQSonBYPBYEhBWMlxf2UNJVbtApQbOyhx3oPmGl6ToXGCdJ6PJl1zWjl13RCdM39h/tYT9524+MQfT8wBgMz7M79Omtv/Hukqs4ovXFG4sfSG0iuq/1E9t35Jvcvb4I1P+9+0ysQbEgttVltR05KmTOtBqyn1b6kc38aTthVt2SSLIOFXCYCvypbYfqprlS0xbuq2ylYU0dPnpbvr254WIV8MYBaADnTGTGUIvQhZKlgP9PDHiRlXuB9ZSrizuInBYISCCegMBoMhHayEew8ErOBWWlJERGnzHvBkUKD7nOMyKACoVHfyXV3oovs8OfmaDqNxnKuqakR6KBc6pZRQSo3+koAJAMDz/EiHw5Hd1fmwYcMGdXu7FkOG3Aqb7SB27nxhWFlZWXJcXJx1zZqv9TZbnre09HWPIFBBEChvt2vUS5d+1nLZZZeuKCwsdMjyYoWPVJ/j/n5evABK/dtGAAnoTAxNh8+hbvM/ftx/a4c8DnSpiSUHOmQah5VwZzAYjIGDlXAPDVXogmtAue/5gCL2POf0nDPrz1lra5+r/UXJ3JIbxJ7o4n6aLI1LFady8q28CQD0o/Xt3R/VB6VUI5ZRFwQhmVJqcDgcl5xRZYuDMWtpFk7MOQG+g0fcL+IQPz8+j1Jq7VjTIbSvblcl35Z8JHlucgUhpIPW0ZzmF5sLdVrdF6nzU09I+PLEGl0XIeej051+NoIvQpbLgS4lrAd638YBYsjtLggCB/kqkTEYjCiECegMBoMhHayEe2iUmggbcDF5gBjQ9/rZZ5+d0N4uxBPSTh2OZ/QAQKmduN1a7YIFz055//33tgS6z02mCS5CgOTkuc4ffnjxrG+//dY9YcIEEqxfHnwlAU9bCCMIwnBCSBt8zocWjuMqXS6X9ZVX3ppgt18c19FR0Ob15vAWyxuJCxY8d+Cxx/5vT06O+fzk5EZN19bdBkOWR6XqWnFQkfQ1kG4DsNe/cQDy0CmoT/DvUwNAB5+wLlUgHSulAmMp4SSOFTMl3Hmel6QUIYPBYEhEuL+/So0bAOXOncVMAwD1UlJyTck8sed5/EXxTcazje+U/rb0dyW/LJk3bv+4Nzg9J4g9zz01nqSU36bsbdvYNqbqr1Vzsp/KPqwZolEHiZfEW13XMXmeH9e1yhYorLWP1w4ROoR8UAjWNVZPwjkJa1OuSzlpuNrAqQV1TupvU6vEY+Q9mnfUMMRQmvKblEo5X69eIPf72pdrTi98jvMyAF8g+CJkOwCxyoA+Imd6JrFWwj3WBPSYEev9i9NY3MRgMLqFCegMBoMhHTxCBEkBDjElJkQAZSeDlDjvAUWr1fKZmaYGYAMAQjiOU6lUnIrjElWZmVkZDofj5999911afb0qPzU1Xa1Wr8vlOKiSkgQ0NGRi+/av88866ywKwBaQ4KnnOO54YJ88QojN6XTeq9Fo1nXtgb548eLRZWV2vdl8Y6vHQ3lADb3+RufOnU+MtlgsB957760tQU9+8BPJIFrq74YA4IR/+xK+xQ+F8CWGxgBIAfAQfIkj0WnREcHxYyGBEksJJ8C3YMIrwzjiWFL3QJdzQQCDwWD0l3B/E1kJd4Xhb0mktHkP+KIBoiY0456MYv1ofat5hrlZEASTYapBU/BhwVbnUWe6G+5p1EHjLIst+e4Wd3zKP1J4w0TDWbrLdOqmhU2of6l+fNrf0sCZODdObz9V17W/uNvtPlcQhPy4uLglgedABQp/z/OctNvTNmfck3H0+HXHfyH2RE+5LuVkoHguknrToHeeR1PMBHS/CHmM//Hb4VuELMZM1YjsNSgT0MMfJ2bKqss5FivhzmAwQsEEdAaDwZCOcAJ+xZZw98PKESoHSqm0cRalFJRSnSAIcZRSs3grOh8effTRuEcffdTgdz50XS2fyPN8htvtdmdna90ez3d2nt/t4XnB4/XyntRU3nv8uKfeZDIVh+ovTint9vu8ceOWUZSCtLdfZw68X6VS8StWrBhx7rnnftv3V0BRRPLDZAVwwL/pAGgBlMOXHLrKv08DOhNDleh7kM1KuPduDMgwjjiWnA50ScdiDnQGgxFlsBLuoVHy3JU6b0mhlHL+MupnVNailJpNvzPFUUrNNpvNBH+uQj1WjbixcfB6vQ5CSIfpYpPLMMVw0jTZ1EgI6dCkaayZ92W6OjZ2xJnTzT+F01/c4/HwAa3dOu+v9ugdPzjy025P+yr74exDADD8g+Hrjl93/BdtG9qGpVyXcjLSrwkjJIGLkEsB3ARgE4AsAOcAuBC+XumRWIQcSzETZBonVh3ocpRwZ3ETg8HoFiagMxgMhnSEXcJdwSKyUpNBSp13n+bsF8bF/uJnJHgC+4sD0HR5ugenOx8aROcDIaSD4zjRAeEghOCSSy7BJZdc0t95dsuf//zAN0ePHv0p2GMzZ86sDXb/IIdIsDCipwNK/b0hABwAtvo3A4Bh8PUALALwM/g+U4GJodZeHl+OhEMsjCF3gkbOHuiSO9AJISwRxGAwogUB4f0uKnnhrVJjB6XOu89zppSq/b3Fgy4oBiDeGrs+Fb4qW2KcVMdx3PGAClti3GQVrzFMk01nnkA+YLyj66F7jzZP6xy1edQKdYL6VIUglVnFD/9w+DpOz7EqO52Eipuk7lF+GMA3/r+z0Vnuvb+LkOWKA+S6/mcC+iAdi5VwZzAYoWACOoPBYEhHyBLu6BTZlZgYAJSbFBnw0nwDwBlz9jsfTJTScITxrq5uJyGkA4CVENLKcdzJriUBOY7r4DjOLcfkwmXWrFmNs2bNagy9Z3SwevXqjL17D+Cvf/1fLj4+PlKHHejfhMDPqgPAT/6NAMhAZ2LoMvg+l03oTAydgE9gH0iYY6NvY8n1uyx5CXfmQGcwGFEGRWdM0NPvo2JLuCu0lDnAYibfHSGqbAX0FzfjzP7iPHzxkriYuMp/2zVusoWqsiU3geK5iCpOFVXXNy1rWjLdFW5zxh8zSuD/DjuOOEyWty0jc/+R+z1R9/s1H+jfBRpwW+3fAhchDwcwAZ2LkMvRGTe1yH2yXZArZgKYgN4XJI+ZgFPVA6Pqd4XBYMgLE9AZDAZDOgSwEu6hUGQyyF+mLmbnTSlVdXU+ANDzPD/SbrdnBwjjRpz5OgQ6Hxo4jisTHQ9dnA9y9StmdIPValW9++5nI1pbDfjggw9y7rjjjoYIHr67YFnqoL2n7yUFUOffdsCXoBwKX2JoNIDz4Au+K9CZGLIEOX4sJFCYgD6Ix/L3QGeJIAaDES2IDvRQv4+KjBv8KHnuoJSK8VNMEazKFs/zwwCo7Hb7ryNZZYsxMNj22DLaNrWNpQIl5tvN8NR4VHX31c0EB8FT6zmszdM6JRxeDgd6dwQuQgZOX4R8KXz5r2b44qUSnLkIWY44IJbiGbkFdDmqQMgyJ1bCncFghIIJ6AwGgyEdvRHQlRrVKjUZFJXzFgRB24PzIbAkYNf+4uJ3wQSgze8Wt3bjfGAl+aKEt99+u6CykibGxY3HJ59sGTFv3ryDcXFxsRB8hhuouwAc8W8AkIrOxNBMAHMAtKFTTC9HbInbsZJwEseSU0BnPdAZDAajk3Djoai8fo4QSp174GdjULmje6KfVbY88LVIMkRTlS1GcHIezzlAPVTVtLxpilfjNdp32PXgcGLIS0O+jJB4PtDfi3DHr/dvX+P0RcijAJyLMxchBzrbpSKW4plYdKDLMifmQGcwGKFgAjqDwWBIR8gEeYBYqMSECMCSQQOO3/lgCLO/uLbL07043fnQ2J3zwWaz3atSqX4wGAxb5J8lI9JYrVbVJ59sGcdxlyE/fy5+/HF38rJly4b+4Q9/OB6Bww/kb0J/xrb4t13wuYSGoFNQnwzf/wl2/20mfE52qYiFJE3MlQj0I/lrJwgCc6AzGIxoQizhHgrFlnAHi5kGBf7+4nHBFhTDV0JdjJuCNAcPr8qWy+U62+v1zjGZTG/LPD2GBBCOIPfJ3H0Vd1SYOtZ35MED19CXh36pH663R+LwETjGQIwdziLkDv9j+fDFTK5+jNcTcsUzUiOXgC7+HyyXgC65uYLFTQwGIxRMQGcwGAzpCNuB7l/1qESUnAySdN6UUtIL54Oqy9NdAf3F2zmOqwnmfCCEuGKpJKDb7SZarXZQJeoGK2+//XZBdbU6NTv7Wpten5rGcRcLn366cexNN91ULrELXQ5RNRLH9qDTQQEASQAKAUwHEA/gLgBWAKX+fUrhK3UYCZgDvW9jyVX9Qo4e6HLOh8FgMPpLYAn3nlBq3AAod+6yOND9VbbM3cVN6LnKVmBVrUhV2VLaex3T83Uec5rcJ93J4lL01rWtuZn3ZR6TYWg5rtcjcfxgi5DHASgC8HMAFwGoQmdsFalFyKwHet/GgUxjydkDncVNDAajW5iAzmAwGNLBSriHRqnJIKCP8/b3F48LdDyIiR6c6XzoOoY9QBi3cBxXEcT50CFhf/FB+147nU5u+vSL50+dOvn4yy+/vEvKsQTBF59xXHSumxHd54TMpjpdFg94kJJyrbW6+svUSLjQKR3QNQxSfUZbAHwHIAc+l8WX6HRaFMH3W1iNzsRQDfqeMJArGST1GyW3w4GVcGcwGIyBgZVwD42S5w70Ye4BVbaCtZ46zTmO4P3FrYGO8a7CuD9+chBCInr9EOnjRQrBLnDHf3P8ssyHMnfFz4hvEu+v+H3Fz/Sj9JbM+2URhKMOxxGHqfK+ypngIGS/nN3asarD1bS8aQoAREhEH6jPi1S/R+Ii5Db4YqQP4ctrDAdwIYBLELlFyLG0IDgWBXS5SrgzBzqDwegRJqAzGAyGdIRzEcZKuCtz7mcEApRSjZjgCeF8MAQ5VmBCpyYwwRNYEnCA+4sPymSQyMKFC0efONGcW1+/OfW++0oOjhgxoldl9Xbt2pW4cOGLUxYufGF7VlaWp6d977nngak6nZZfuPDZb/t31gPDypUrc+vqaLLH86OmqupBQ3u7Hi0t9niXS0c+//yr0REq4x4KqT5PciUfKvzbZvgSt6KYPhU+l4UDpyeGrL04tpzJIDnGiJkEjVxj+R3oLBHEYDCiBdGBHs5+0bn6sP+wmEm8o7PKVihhPFSVrTaO46r9MdNpC4pjrcpWJHDXuPWOI44hpfNKRxcuL3wrfkZ8U/md5dOaVzTPTPpl0lcAehSDnWVOQ+ua1rxA0VhwC6Tl7ZaU+Hnxp/ajAkX1Y9VF5unm2oQ5CQ3i/ZZlliHeFq8u2oT6+kX1E8BBGPLSkC/5VP63yXcmW2g9rWlZ2TIxaW5SpW6Irr990Af6gyr19bMVwGH4FiKrAOTh9EXIwOmLkKt7cU5MQO/bOIA8jm1ZKmqxHugMBiMUTEBnMBgM6Qi3FCHAkkExid/5oA9SEjCO5/nhNpvt5gDnQ7D+4oFCuKUb54N9sDoVgjAo32un08m9++7K8ymdQZzOffqnnnpq8tKlS7f35hjPP//ilN27q8cuWrSo+amnnvref/cZ8y0uLk7Zvbt6BMdRumvXrpKpU6e2RmQSMnL22Wc3X3NN+V5KKTEajakZGRnTq6urD7rd+ba0tLRI9PMDuk8ADMrPUC8IltzoALDfv3HwudTFxNBc/z516EwMVSF0MkZxhU8AACAASURBVCEWkjSxKqDLUcKdg+//EAaDwYgGxB7oPf4f77/ejfbrgL4S83MPVmXL6/UWAoDdbr8egCnMKluNHMeVy1xlK9IMuvdaP1xvH/n5yGXHrjh2U+m80puNE4wl1m+sZyfNTdo67M1h20I9v/H1xrOaVjRN49t4Xc6jOT8IboEc/83xS50lzlzdRJ097rw4AAB1Us5V4Urq2NYxAsC2hDkJDZZlliENrzdMNU02VVCBgnCD7uXpltyncr/lW3iNrkDnsNlsp3qiOw45jkVAPA+FlNe3Ur8JweIAHmcuQi6EL2Y6D8AM+BYhlwEoQXiLkGMlnolFB7qcJdyZgM5gMLqFCegMBoMhHTxYCfdQRGUyiFJKKKXGMEoCxuHM/2vd8K2eFvwJnZpAp3jArTPGnA+DVuRfuHDh6IYGa5pO91enx7NOs2nTvyeXlJTsDdeFvmXLltR9+06M9HjGa9au3T7x3ntrD8XHxwfd9513VpzldI7WUerG22+/d9bUqVN3dnfcqqoqfV5entSJlV5TVFTUUVRUdBAAPB5Pgcvlmm4wGI6oVKrmgT63CDDQvQIF+ATyKgBfATCiMzE0Cb4e6m74EkOioN7W5RisB3rfxoqlEu7Mgc5gMKIJVsI9NFE7935U2RIAuADf4glCyGCtshVpBm3MZBhtsI38fOSyQ+ce+h/rN9azjZONPw57e1hxOM/NfTJ3n6fBE9e4tPEi6qWc/aA923HQUZi5ILNcP06fKu7HGTmhYEnB9orfV1xQ82TNhW0b205Yd1uHmSabKvJfyN8TKJ4HE9MHm8CuTlB71QlqceEGAQDCERjHGXtTXaonBvrzMpBxUweA7/1bXxYhx1JFrVgU0OUq4c4EdAaD0SNMQGcwGAzpCBnIBwT7gyfKkxfq7zk0KKCUcl2dD8Fu4XM+dK0a4AhwPjRxHHeiq1vc73zw2Gy2OzmOqzYYDGvlnyUjENF9DlwOlWqsQEiOy+l8w9AbF/qSJa9PdLlGaFNSFra0tl4bv2jRorF/+9vfztivuLg45fvvLUMTEu62CoKTfPvts4W7du06HMyFvmrVquwFC16a8cgjD3x11VVX1UVgqlITyeC2p98EOYN2Kejt750dwA/+jQDIRGdi6HL/fRZ0JoZOROxMQxNLArosDgc/cpRwZ4kgBoMRTfSmhLtSc1iDSkDvocpWsP7ifa6y5Xa7x3o8nl8aDIaPCCE9tkiKMQbNe92VuhfqJor/dh51FrRvbU8J7IneHYQjGPrfodvKbyknlrctFwJA9sPZa82Xm0cIgpAauK8qTsUXLCnYfnT20V9bd1kLVYkqW/4L+XuIurPqWuvnrRkN/22YNPTVoV9qMjVuAHCWOI2VD1RenP3X7G/izo9ridysI4pcMZP4eLQ70MOlp0XIZyP4ImQ5F9DGioAu5sDkEtBZCXcGgzHgKDX4YDAYDDkI24Huv2hTIrIkgyil6kC3eFeBHJ3OB2OQ87MFJHjqOI477hfCuzofenPRPaiSYDIy6OYsus+12vtcAMBxiSDkNmHTpn9PPnr06N6UlBRPampqtwk70X1uMDztUqnSBZXqV/zatcuK7ruvA8nJyaftK7rPU1LOawcoTp5clRbMhS4IApYt+2BCTU1y+rJlKyZcccUVdRynqJ+IgRTHpU4+9Of4FECtf9sOQA9gKHyJoTHw9U8XHS4F8InpIROafSQWHehyudfkKOHOHOgMBiOaCKuEOwbZwluZkStm6k+VLRchxApA7CkeiSpbYqxMYqwyV1Ryquf53KStWY9kfSuWcxd7oov7decCp15KvG3eU/G2p95j6m6slo9bcsV/C1ZB3/5le1pgT3RVosrjqfMkld1SNmfYm8M28h28uvyO8ssggFMlqJS02GKgkVqg7+vxw1mE7PDvWwhf3CRFiwe54hm5FgPH3AJnJqAzGIxQMAGdwWAwpKM3PdCVmhHoczLI73zQ9eR8QKcwruvydB6nOx8quzof/Lc2qfqLKzABOCgdwx999MkUntdwHs8dWo8/1UKpizidVH/fffddotUm8UuXvrSpOxFddJ8nJ89pA4C4uN/ZW1tXxn/22We4+eabT+0X6D4nxCeGx8dfawvmQl+1alV2SUlHnsl0g+3w4XfzP/vss6y5c+fWSvYi9I+B+hwPys9TmETq3J0ADvs3AEgDMALALABjAYwH0Aqfw6IEvn6B7giNHYsCesyUcBcEgVBKo6XPK4PBYIQtoIexT6zSr7kHVNk6FSeJAjn88VKAMB6sv7gojDdxHFfRJV4ShXEpRUslve+D8hrXccgR17K65YKkuUlbxbLtqTembmt4rWF2zZM10+JnxH8GAG2b2tKaVzaPzFuQtzugdDnEnueOg47C7L9kr7PuseY0Lm28CGloTrwx8bSxTvU8n2Iqz/lnzt4T95yYXvNkzYXw90QHAPN0c3P+c/kbKv+n8tKSa0uupQLlOC3nKXi1YL1hjCFS5dEZ3SPXdzIS34fuFiFfBCAVwA3wiecV6HSnR2oRslyvEyvh3keYgM5gMELBBHQGg8GQjnAuwhRfwh1d5u53PhjCcD6YEby/eKAwXtdNSUDHALsYBmViREoGq2tk3rxf7SkpKSnter/LVaA+etSSZbcbDO++++7Q+++//1jXfXbs2JG8b9+JkTw/Vt3S8miCeL/bnaRdsWItrrnmGlV6ejoAYPXqTwrtdp3e5XpT297+pgkAKBVAqY58/PGqwqlTp+4FfO7zd975aILHM1mdnn5rU339jylvv/3BhKuuuqpWQS70noJlOcoFSu1Al4pG+JI9swCsBWBFp9NiCnz/35xAZ2KoIfhhwoIJ6IN4LEEQOMjnqGcwGIz+Eu7vlYAzWygpAv+C3jOuIfz9xeO6W1AMQLztqcpWByGkNiBOCoybbL2sshVRpFrIzOg9hjEG6+jNoxcbi4wd4n3aHK1VN0xXZTrbVEUFivYv29Pqnqv7mbZAa+E03Gnf65N/OTnFcdBRmPW/WevS70ovS7szrbT89nJYFlnO10/UO0zn+8zogl3gmj9sHn+q57maULEnumWZZVz8rPgtorvdPN3cnPlA5rbaJ2tnAUDugtwNg1w8lzsgluP6NhpLxIuLkIf4/16JzphpFoBL0bkI+TiAcvR9EbKcwnYsjSOOJVcJdxY3MRiMbmECOoPBYEhHOBeW4j4xnwzyOx9MgYkdAHpBEIbb7fa0Ls6HYP3FRedDC8dxpxzjgSUBOY6LlLtSapTqoBl0c37ooYeOBLv/6aefHvP991VDNZrzPOvXbxx9ww2W8q4udKPR6C0qyiv1eJpVQPNpzx8/vuicwEUDl1/+i/Lc3APtZ46Uj8mTJ1vEv0T3eXz8b62EcIiP/631yJGH8ga5C50RPnIJ9DyAY/4NAJLRmRi6GL7kUAc6E0Nl8CWTejMOE9D7BivhzmAwGKfDHOgBBKuyxfP8UADEbrf/soswHqrK1gm5q2xFGCVWaxu070ugeA4AabelVQBA0/tNk0rnl6Z4m70m3VCdZcjCITs54+kCevb/Ze83TTXVpPwmpRro7IneuKYxQzdelybuxxk5oeDVgs2aLI1L7Hku9kQXnAIXWBreWeI0NixpOE/8u+apmmnDRg7bKPZEVwgD9XmRqwe6HHFTo3/bCUADXyusrouQK9EZN9X3YQypiUUBnZVwZzAYgwImoDMYDIZ0hFzFGJC4iNqkgL+/eDjOh649zsSedoI/qVPPcdzxbkoCxuIFbdS+531k0CaDulJdXa3btOngKL1+nispaXZHXd2O1GAu9EmTJrWvWvXhxq7Pp5QSm812jkajOfW5nT17duPs2bMbexpXdJ+73Werk5ImuCj1wmCY6GpvLzJHgQtdzpXoUo4nZ1JACrp7fZoB7PFvagD56EwMne3f/yQ6E0O1QY7RdRwmoPcdqQV0jpVwZzAYUYQABQjo/ayyxfuPYR7kVbakIiYn1ROU0kFbwSuQtNvSKuw/2tMcBx15AJD/fP6uruI5AKiT1F5RPBchHIF5trldEIS0wPu1edozFnWq4lS8Kk51KrZyljiNYs/zwuWFH3mbvdrK/6m8VOyJPohF9EheA4b6gESzA10Ogr0+HvhaX5X4/06Gr0f6CAAzAMxE7xYhyylsy+GijqmqXcCp1oqxmG9kMBgRggnoDAaDIR3hXOwN2hLugiBo/WJ40CQPOoVxfden4nTnQ1UQ50MHx3F2u93+e47jSgwGwxfyz3BAieZAsz8Mus95MN55551CiyUpISfnF40qVRzVaGZ71q9fHdSFHkl2796dVFnZnE7ID2houDRFvJ8Q0IoKe8bevXsTzjnnnDapxu+OI0eOmPLz8x1Go/GMoNwfcDLCRy4Hek9jeOFL9pQB+AJAAnyJoeEAfgafQ90OoBS+xFApAFuQcWJJQJfF4YDOOUneAx0sEcRgMKKHcBcUD8oS7sGqbHUjjPe5ypbH4xnr8XiuMhqNy6LEOR4plDTXrkTFos62TW1pzkPObPHv2mdqx+U8nnMg0CkuBdad1lQACOx5nv9c/oaqv1TNtO2zJSX+IrE3LuGI4Ha7SWVlpWH48OF2ucceAGLFgR7q+M3+7Vv0bRFyrDnD5YqZAPkEdBVY3MRgMHqACegMBoMhHYOuhLu/JGC4zgdNl6d7cLow3tCN88Hei9XygzIRJgNR7aCJZQLd5ypVHAWAlJQr2+rqNqV01ws9UkyePLn1z3++dYvT6Tzj+sxXLr4oSAl4aamurtbdcsu9V06ePLri5ZcX7ZJ7/B6IVge6XMmN3ozRBmCff+MA5KIzMTTev08tOhNDJyHP75fcDnQ5XBvi/3fMgc5gMBidiCXcw9lPtuvnflbZsgXESZGoshW46HrQi6oRJOqrtfWBqHl/2za1nep5PmThkJ1Ny5vym95vmlT9WDWkFtFTb0ytTP5V8slAt7t5url59BejVwZzwMvBSy+9NHLHjh8KFi3659a8vLxgruRY/BxL3QN9MDnouy5CjkdnzNTdIuRYE9DlLuEuV3zGBHQGg9EtTEBnMBgM6QjnYk8sY96vYIpSSgRBMHXnFu/ifFB1ebozwPnQxnHcyWDOB0KIS4IyckoWkpU276hIBq1duzbHZiN6l2uVtqpq1WnlBLdu/Xbo/ff7ekpbLBZNpN3oWq2Wzps372Qkj9lfXnvttdH19YbU7duP6A8cOPBTUVFRR+hn9ZuevhtK+970lv4mm8Qef5UAtsAnCoju9CkALgDggu//kQz4EkdSLeyIxRLussyJOdAZDEaUIVsJ94D+4nF9rLLVESCMVwUsLA6Mm+yEkEgn/ZUoJAei1HkPajq2dWSL4jln5ASxJ3rr+taRXotXq0mXtox6MKF8oMTz6upq3ZYth0fU1SUkrVjxQcFDD/3PkYE4jy5IeX0bC9/J/s6hHaEXIbf7H8uHbxGyVJ/PWBTQJR/L6/WKnwEWNzEYjG5hAjqDwWBIR8iLvYDkSlAXNqVU1Z3zAUBgoseEMwMAW4Aw3shxXFk3zoeBdKkpVUBX6rwH/Zyvuuqqk2az2R1sUUt2drYdAH788ce4W265e+68edd++6c//emo/GcpD9XV1boNG/aM02qvd1ut24z/+c8bY3twofcruKUCRReXCg1yHwDA4/Fg+fLlBddcc02V2WyOdLAb7Q70SH/HbAAO+jcCIBu+pNCFAEb5twZ0utMrEbkERCwL6JImd/0OdJYIYjAY0UJY4jAhRKCUdhczgVJqDIybgi0ohm8xcbAqW4HCeENXYdwfPw1kf3GlCugRWWwepQz6agM5j+ccoE7KBYrWabelVRgnGpvLbi6bnbcgr9g4zmgFAL6DV5XdXDYz+brkH7v2Qo8FVqxYUWCxpMbHxV1oLy5eNfz666squnGhR+Q9FQQBra2taoPB0ON+ZWVlOrfbHTd69GhrJMYNQjQ70CP5HetuEfJU+BYc3wLfIuQydMZNkVyEHGu91sWxJJ2Tx+NhAjqDwQgJE9AZDAZDOkQ3xZkP+PqLxwmCkAgAPM8PdzgcGUGcD10jIrG/uJjQqQ5M8AQI4zYJnA9SoFQheVAnQyQiKuacmZnpDuUCX7Jkybja2ri0Dz9cP+mOO+44Ho6I665y61vXtWan3pRawel9SSbBLnCWty1Dk+YmVWkypXVo9IXXXnttdHNzQnxa2vxWqzXTsH37C6ODuND7/f1t+qAp17rLmpX3TN53nJajAOA85DQ1L26+IO3/0vZoMjWuwPG+/vprLFv25VS73a65++67S/o7/gBw6rvgdDq5++9/aPqMGT+rmDdvXmUEji1lsokCqPZvUwDsB1APn6A+Ab7ShR4A5ehMDLX0Y7xYFtAld6CzEu4MBiOK6DZmEvuL+xcSJ1BKjU6nc0Y3wnhXcT2cKlsdHMcNumuwIChaQFcYUTNnwhEQ45k5B87ACd5mb3z5reVXD1069FPdEJ2jdH7ppa4yVxbhyA89HZMKFNV/ry4yTzPXJsxJaBDvtyyzDPG2eHWZ92dK1k6rr4juc6PxOmdq6qz2ysrt6VK70BcsWDBm584DZ7/88oJjSUlJQfdxuVzkgw++yGhvbx392GP379Pr9ZHMD8n1W3Tq+1BcXJy6a9fezD/96Y8/abXaSHxPpLz+FxchmwAkA3gbne70K/xjN6IzZjqB/om4sepAlzSn6fF4xOsGJqAzGIxuYQI6g8FgSAd1uVwjBEHIDZLg0QbuKAjCWADtAUK4pQfnQ9QE1WGgVAEdUO68o5off/wxrrj4x7Fa7fXu+vrVKa+++uqIhx56KGRyxNvs1TiPOlPqX67XZtyTcQwCUP9S/ShPg8fkbfbWDzYBXXSfazQ3eVWqRMFsvsJWX/9hSggXep/gdBzv+MmRXfVQ1Tl5z+Z96ypx6S1PWyYSHWkkmtOTcg6Hg1u7dissloykjRt3nzV//vzy+Pj4SAqFsjrEV61alXfggL2wvn5j4hVXXFEdQUe91P9PEPjE8p/8G+Ar6S4mhi6FT8hoRmdiqML/nN6MAciTpOFkHAdSj8UEdAaDEWVQr9eb6vV6xwiCoEWIKlter/ecKKiyFWlOObEH0AU/kChx0lE7Z8MYg3XY0mGfld1adlXZTWXXcnrOxbfy5pzHc9Yl/yq5pqfnUifl3JXuxJptNcMBbE+Yk9BgWWYZ0vjfxvOMU4zl3VWoGkhE93l+/swGlUpPTaZZjuLi94O50CNy4jU1Nbp16/aMaW01J3366acZN998c9D9Dh06RE6c0Bja2jSp27dvT501a1ZD0B37hywl4p1OJ/f55zsKKytVKVu3bm2I4FzkiJkogBr/tg2+9iDDAIyAr9T7+ej/ImS5hG25YiZZxgpwoEeD+YjBYAwQTEBnMBgM6aBer3coOssCdnAcVxMgincQQqxOp/MParV6rV6v3z/QJzwAUIWW5FPqwoGon/OSJUvGWa1ZppSUPzS3tjoTP/543dl33313SSjh01hk7Ei5MeVo09tNo+oW1o2GAOJt9hpSb0k9YhhjkKqkXp957bXXRjc1xSUkJ89o83obOQDQ6eZ4tm9/I5gLvV8kzU2qBbCn/pX6c8tuKJspqIQ8VZLKkvZo2g51ivo0wXXDhg0pJSVuJCTc2l5buyjl/fffHxplLvRTyQ2n08l9+umXY73eCaqamp9SP/jgg4Lbb7+9NALHB+RLBgVS79++hm+R2FD4xPSRAM6Fb2X/CXQmhhrDGANBxpGCmCrhLggCK+HOYDCiCcrzfIrX6x1LCGnvrsqWx+OZLAhCflxc3OKBPmG5CVhAHfXX0r1EifOOicXyhjEGa8HigrWl80vnCVbBmH5X+lc9iOen3l/OyAlDXh6y48Q9J6bXPFVzQdsXbSdse2zDjFOM5fnP53872MRz0X2u1V7u5nkHx/MOxMcXOWpq1qdI5UJ/6623hjc1JSYaDBfx69Ztyrrsssu0GRkZpy3G7ujoUO3adZio1WOp3V6j27Jlb/4FF1xgiaALXeo34rQ4YMuWLeknTmhTvN5hqi+/3DNkxowZjRFwocvxYQoWYzgBHPJvQGQWIceqA11qAZ2DLycZE7+7DAZDGpiAzmAwGNIhmEymL9DpzjsD8TpNqVYCKFdIVuIFetTPWXSf63QPuDhOD7P5xo76+vVhu9BNE03t1ElLmt5vGgUAqTenHhb7Ag42vvlm7zC1Gnx7+x1xgfdzHKf6/PPP84uKirr+rvXr/U2am1TrLHUeaVvfNobEE1X6k+kHu4rnHR0dqs8/35bldE5FaemJQoOh0C6BC12OpAAFgI8//jivspJkpKXd1tLSsta8du36sdddd11FP13ocv2ehnqd3ACO+jcASEFnYujnAGbD1/dPTAyVwdcXsOsYCDFOpJCrn58sc+J5njABncFgRBGCTqc7rNPpvgXQ2t1OXq93HJQZNwDKFJKBGIgflArfwauq/149Xfy7aUXTlPjZ8dXhxD6qOBU/5OUhO45eevSX1t3WYepEtW0wiucA8NVXX2W4XFBT+gXX0PBFoni/VgvPd99ZcgF0jRH79ZmuqanRbdjw3Vla7Xx3RsZVrtraHUnvvvvu8AcffPBQ4H5btmzJqKzkyM6dh1MsFkumWi14JXKhSx43OZ1ObsuWfUMIOR85OT9rKSsrSdu6dWtaBOYiR9wXzhiRWIQcqwK61CXcSZS0vmQwGAMIE9AZDAZDOkJeWAbo5oMvGpQHJQvoSpx3VNPpPr+2GQA0mjyekMsQrgtdsAtcx9aOLPHv9i3t2YYxBqvYE30w8dxz//qysrLSGOyxGTNmWAL+7Pfn+I033ihY+sqyCxalL2w1wQQAaH6leWTaw2kniLazZcWHH35YUFGBuMbGFFCqIXZ7oeHkyaPpUeZCJ/Angj79dMtYSi+ATjfEk5x8bVtNzZeRcKEPpAO9J5r822744o8h6BTUJ/mPVYXOxFAdZHJr+5Gz7CGkHksQBALWy4/BYEQPFOFdGyv5+lnpArrS5g1EyZyDlVT3tnlVZTeUXeoqc2Xl/iv3M8MYQ3vZrWVXiT3RwxHRWz5uyRX/zVt5Q/um9vTAnuiDhWuvvfZkXl6eLdhjaWlpXReH9gun08ldeeU1v25uTkwcNuzqarXabNLp5vCbN78z5oYb6o+LLvSOjg7V1q0/5dvtU9DYeNzI88koL2/I2bJlb30EXehyOdBPuc8zMy9s0+tTvZRORIRc6HIJ6L2hr4uQ5VwMHDMl3HmeZzETg8EICRPQGQwGQzrCvdgTKKVc6N1iEiUnwpQ470E/Z2+bV215yzIs4dKEasMogw0AqJeSytcrh+786uA4wIDm5jlJ4v6UCqSlxR2/cuXKvFtuuaWiy+FOzVewC5zY8zztzrSfqJdyTW83jap/uX5kxj0ZxwaDiH7gwAHz/ff/adZTT/1ry9SpU1snTpzYLvWYXq+XvPrvNy8+WeXI+Uj1kfDPdf/8sHV/60WWFy2p9X+rPyfjiYxviZbQjo4O1bp1X5/V3j5Sa7GUABhOgSauqSkhOcIudDl6oNOPP/44r6qKZCQn/6odALTaTJ7jLhEi5EIHBp+AHogXQKl/2wggEUAhfH0Ap8OXHLIBqPXvr+vXmYZHTJVwZ8kgBoMRZYQroAvoXIikNJQsJAPKmnfUuO7r/l030l3tNuc9mbePqH2LXlvXt6Y3vNowyXXClR7Y83zY0mGfld9afmXburY84zjj4Z6OK/Y8N51jKs/9R+6+E3/0lXOHvye6DFMLyf33P3CuXq/3PP30U/tnzJjRFObT+vU5fv7550dXVPB5KpWTB9QUANLSfuEsKfksMdCFvmXLloyqKn3yiRM/cF6vEUAqbWwsTTl61JQZQRe61It2CQC4XC6I7nO9PtULABkZF7WXlX0fKRf6YI6ZgPAXIccD6IjAeKGQS6gXx5K8hDtzoDMYjFAwAZ3BYDCkI9wLMSWLyEqde9QkRiJIVMyZ03ECUROheUXzsOTrk8v0hXq75S1LAakl5vtvumNHi77ljO81x3H08ssv766nHwDA+q01ydPoMaXeknokwHVxtOmdppH2/faEuPPjWiSZUC94+ulnzvnxx4YRzzyzwLZ69aqNcoz5yiuvDK+utaZTXEA/bt4Yf7/3fhI/Lb6dt/N1Lf9tyXGfcBt1I3S26upqA23mzW2tn+t8ufPpAErhcpbqXY2FCVVVVYaxY8dGrC+7xNC1a4tHOxxGPc8v6VxkIThUdXWq5NWrV+fdeOONFX089mB1oPdEK4C9/k0FIA++pNBZ/sdvB1CNTqdFdQTHFpFbQGduCgaDweiExUyhUaSAHtD7nTEI0aRr7K2ft46peqQKeU/m7Wvb1JZW/2L9+bpCXUPBawVbtJnaU325DWMM1pHrR36kTlJ3XfB62nss2AWueWXz2MCe52JPdMs7lrHxs+IbBrqU+86dO5M++2zHORwHYd68648XFRVJHoM4nU5u+fJPzheEKZwgHFQdP35hXlJSgcBxnApw2Xfs2Ff44IO+ntoNDRaD13ucq6s7RAi5mBKiFXj+W/XJk/q0xsahBqnPNZLs3r07qaqKS3a5KtVlZa+mifc7nSbtV199m9dPAX2wlHAPl+4WIQ+HL346C8CD/seP+2/tERpbJKZKuHu9XjkXBDAYjCiFCegMBoMhHeFeWCo9GaS4ufuTQYqbN6JgzpyeE9JuSytrfKNxWNP7TYWcluMFl6BOuSal/MZJN/bZkR0/I77JOM7YHtjX2zTR1K7L0x3o2ut7INi3b1/8jh37JwjCFG7Pnu/Gbtu27dsLL7ywuReH6HUg7fV6ybJlH0wVuIsFY9ITre0dP49bsGBB0RNPPEHiLolrMk41HuJMHA8Ao0aOss6fdGXp/cceH8GRf4Jwt3mp0EgE+nO1mndwY84aE6nElRyr9jFjxpTS0aPr6wFHl4eHYNy4cf1ZTCFnD3Qp4AFU+LfjAG4CsAlAkmFXuAAAIABJREFUNoDzAMwA4ERnYug4gJBlQMNA8hKBAeNA6rGYgM5gMKIMVsI9NBQAKKVKm78iFw74GfRzTpmXchIAGpc2Tin5ZUm64BC0+uH6hvwX8ncHq64VRDw/A87ICQWvFnypydK4RKFc7IkuOAVuoMVzAHjppcUTnc58HSFuLFq0aOIbb7yxvRdP79M14GuvvTa8vd1oSk5+oMXp/ERnNK50Pvjg7J/0er2xpaWlJD09/VRQcdttt5QuW/b2VEqHQaX6l0CplhLiESyW/6RMmzatvi/jB0EWB3p+fr79ssuy/X3kAyvixyM5uaC/JfLl+DBJGVsGLkK+E77S7o3wCeoT/PsELkKuQf/FYrliJlnG8jvQWczEYDB6hAnoDAaDIR3hXpwqvRzhwEfBA4PS5h01DhJOzwmpN6WW1z5dO15wCJxpiqneNMnU73LmwYTywSCeA8Czzz43xeXK1MXFvWl1OC43Llz44jkXXnhhOC70Mz7Hu3btSmxra9POmTOnR0fAK6+8MrymxpqhN9zj5Lh0ynHz6Jo1b0158MGWAwkJCRDFcwAgHMEL3yw8i9I8olFdDZ7aOA4GUPySfl/9Ws6PP/1oHj9+vKTuj+3bt6d89NFHo5977rlv+tFvjwCgd9111/FInluX4wPR5aboaQwAOATgG//fOegsW3i1//F6dCaGqtA34TimSrizHugMBiPKEBBGTOAvs6rkmAlQXvwgoqR5R03MBPhEdNt+W4Z9nz0fAPIW5O3pb2sqbY72DGFUFafiVXGqfl/beL1eolar+/wa79y5M2nXrqNjdLrHXZTaua1bnxl34MCB7/viQne73WTDhg0Zs2fPbuipL7nT6eRWrPhsEiGzER8/yWY0Zjra2r6It1gs+jvuuKMJwInA/VeuXJlbVeXK57hrQamXUOolavW1gtv9if6RRx658L333vuiD1MPG0EQ8I9//OPsiy++uGrGjBmWPh6GAEBBQYH95ptvrg21cz/GiIWYSRynHcCX/i0OnTFTsEXIpfCVfO/LOHI60CUdiznQGQxGOCg1+GAwGAw5YA700Ch17lGVGFEa1EtJ84rmfPFv+wF7quOowzSQ59QVx1GHqe2LtrTA+zwWj6ZldUuW4BR6dX0nus9Vqju9HJdA1erfe/bsOTx227Ztyb09L6fTyT3++HPTn3hi8UUNDQ3a7vYT3eeU/pyo1RN4ADAY7nC0t+vi3njjjZwgxyVVJxtyQZrgEc6HIBSpeDpRRfEueF6lev7558/t7bl2Q9BAXRAEvPDCS9M2b/5p8tKlS4dHaCwpiEUBnQbcngRQDOC/AJ4F8DGAOgAT4XOr/xnA9QCmwFfWsDfIkTxhJdwZDAbjTMIVh5UaNwDKFdBF5/1An4fsREu1gdb1remOA45c8e/qf1RPpN7Bee7btm1LXrJkaaHT6exzLvyllxZPdLmG6IzGq51G42/sDkemYdGiRRPDfPppr8unn36avXTppomrV6/O7ulJr7322vDaWj7dbL7VBgBqdbbAcVcKH320frjNZlN13f/999+fJAgajuf/DUG4hFD6c87rnacGPPj++8Ojwp5seHM548v56aef5qxbd2jSK68sPUcQBrU2GWsCeuA4VgDfA1gJYAGAN+Dro54I3yLkPwG4G8BMAAXwtdHqyzhSIoeAzhzoDAYjJMyBzmAwGNIR7sWeAOUlQ0SUmghT6rwH/ZyplxLLW5YCV6UrPunapDLDGIO18Y3GYWJPdMMog22gzxEABKugdhxyJFGeksTLEhs8Fo+m5eOWPKIilHoogT78Y4nuc6Nxvg0AtNpfu+32JaZeuNBP8d577+WfOEEzed6keuONN4Y//PDDh4Lt9+abbxbU1ranC8Jxzmr91anFCTyvV69cuTbvnnvuIVptp/6u1+vpfffdtdpWY5tsbDEWNJ5sPAoAnhxPE02k9nvvvff73pxnb1mzZk32sWOWIS5Xoe7DD9dMufXWW4/30YUuV9IhFsYIJTbbAfzo3wiADHQ6LX7hv68Jne70Cvh6B/Z2nEgiSwl3f7JyUGcsGQwGIwCxhHsolB4zAcqbv1LnHRW0rm9NF3ue57+Qv7tldUt249LGKWJPdKIePD3s7XY7t2PH4cyTJ9Xxe/bsae5lqyoAne5zrfZxFyFaEAJoNDd5+uJCdzqd3Pr1O4dVV6embNiwu/DKK6+sjYuLCyrkrVz5eZHLZdQKwhOnvgeCYOXq672aTz/9NH3+/Pmn7f/3v/+9+O9/f4ISUni2Tje8tq3N3tTaus+cn++qv/rq24LGZpFCEAR8+OGn4+32oYbS0orc1atX5/7yl7882YdDyXV9HgsxE9BzjCkuQhYXIhvQ2Tt9IoBpANwAytEZN7X2YZxIw0HieMbj8TAHOoPBCAkT0BkMBkM6wr0Qo1BuRRClCsmA8uY9aBIoPeFt9aq9zV590tykcrFsu9gT3VXqigtTQJd8rqbJpjbg/7N33uFxVOf+/56Z2d606rIsyZZ7xeBCscEYY8ABLji5QIDghEsuJpBAGjfJJSG/NG5CCCSEEhzAphibEozBYBtsU2Ij09wwbiqWJav37WV2zu+P2bFWq7aSdtfanfN5nnksze7OmaPdHc/7fs/3fQHnv515bY42bagzpCU8ofbr7LW8JfbyhkePHjXt2rVvtiiWCG73PRFCtsB/+umRGZ999tme+fPnd8VwKOrz+bjXXtsyk9KLiEaTE3z77Q0zb7utuSI3NzcQ/eQ5c+a0X3bZ3M9lp2zkpbIYpaWluTzP97p+3nXxXfWeXZ7p+qv11HC+4bXOFzpLgyeDNus3rBWGUkN0M/Hh0ispIEkS1q5dN08Uz9LYbP/rqK+/Ke/ZZ5+deMcdd5TH4/hxJp0d6ANBITvRGwHsAqADUAo5MTQNculCEd291Ssgi+tDHWekJGUsVsKdwWCkGDGVcIeKYyZCiNqFZDXNOyViJgBwf+HOVcRzTs9JSk/0jk0dU8VWUaPJ1/SKAc4Un376qf3UKb2VkHG0rOxI3oIFCzoHKpveF08++dRst9tg4vn3Q37/Lh0AUOojXq9e/8QTT8566ql/fBzDYSgAvPXWWwXV1UJOScm3m2tqnsx66623Cm688cY+hebLL198ZObMurqeMZMRJtM5mTNmzOjVZowQArt9vmfOnP9GKFTc3tbG1bS0fGYG/kUXLlzYMpQ5D0Cf97SbNm0qrKjwjs3M/EGn0/my+ZVX3py1YsWKUxw35Et3Mu6ZU70H+nDH8aL/RcjLIf8/298i5GQKzslyoDMBncFgDAgT0BkMBiNxsBLugxBOBqlx7imTGFEbmmxNMP/H+UcjHROcnpNyV+VWjiYXBSCL6KGukMZz0JMJANm3ZlcJNqEvh23/xzCZQueeO+Owz+fTyPGxAoFWO03MyMgYrEf76e+v4j63229w8rw91NLytr0/F/r8+fO71q5d+2FfB/R4POdHl/ujEoX7A3ehdoLWa1xslAhHJPut9sqONR0T3O+7C/Vn6bsIF7dLSY/3WXGf6/W/8Wo000VKLyEjdKEnkmRdT0ebgB6NH8CR8AYAOehODC0DcAVkZ0UFgKoRjDNUktIDnZVwZzAYKQYr4T44ahXQR9t9VjIZ9e/1mP8dc4iKlHBa7vT7lHVj1in7Cnv9SPugxxOPx8N9/PGxPJ1urpifP9tVU3Mi49NPP80Yqgt91qwZTT6fZ19Uy3EABZgxY3rMwrTiPue4hZLVOtXX2XmBcevW9/t1of/yl7881NdxAoHAhEAgYI3ev3t3Wb7LpdU3N+9Ce7vH3NHhyaFUAiF67Nq1K7ekpCR6AiPh9HuvuM+DwXlCZuYcP8DRyspPR+JCTzTpXMI9VvpahDwevRchn4QcNxmGOc5wSPjfji06ZjAYscAEdAaDwUgcsQaNai9HqMa501TpaxdPUmXOfQnlo008B+Se575y3+mkietjV2bG8ozmoRyjqKjI99prr7070nPx+/1EcZ/rdKVBABCEK8WBXOhDgXAE9u/aj3EWzqII5UQg1H6rvVLySHwcxfMeB4p0n5tMizwAYDJ911Nfv3O4LnTmQB/aGIjTOC3hrQyABnKvP0VQnxd+ziIAFsjJoaY4jNkXzIHOYDAYvYnVga72mCll7qXjiOoWDkRUGxj1EI6AaHuf72gSz4Fu9/m4cWd1arUmSRCmhsrKDgzZhX7vvfceBXB0BKdCgG73eX7+sg4AyMu7tKum5uMBXehD4bzzFjQVFJx0jxtXNLerq6uuo6MjvKA0C9OmTYulslgs9PpOKu5zq/VGFwAYDLMDTucCyzBd6MlyoKdDzBTPcfyQP+PK5zwb3THTpZB7pYcAXAWgHHLZ90RVmki4210URRYzMRiMQWECOoPBYCSOmB3olFJVliOEegV0QH3zTplkUCoQ2fM8+9bsKn+F3+T8tzOvE50YqogeD1599dWxJ09K+RrNWNHp/MAAABpNTqi5WZc5UC/0ocBn8L3c9UQglLf23j9CTn9W33777YLjx1tLJGkicTr/bFb2+/05uldeeWved7/73QpBEIb62U6XXnupJKBHEoSc8FEWP+QC+B5kd8ViyMkhF7rLFlZBLnUYD5LVA50DSwYxGIzUQemBzkq494/qhOQo1DpvxgiJdJ9rtSYJAMaMOcdZU3N0WC70kRIMBrFlS1mp318qeDw1Wo+nRivvHycM1gs9Vs4555yuc845RxHKOwHUjfS8Y+HVV9+c6XLZTZL0MXG7PzYAQChESGWlb+ymTZsKV6xYMZzzSPUe6KleHaw1vO2BvAj5egAFkF3qcyEL3DXojpviuQiZAyvhzmAwRgFMQGcwGIzEEas4rGYRWa2JMLW+52qcc0LwHPDYlJ7ngk0QhblCFwC4ylw5wdZghyZbc7r0uiRJ+NnPfjb3yiuvrL744ovb+j/q8KmqqrIaDH4f8CyhVH6fOQ6SxQJ3VVWNPdbjUBpTjJy0XniCINCiIluDJB0h3dXAZex2u2ukx08AyewXmKoCejRKgnEXgGMAitHttJgTHr8O3Ymh+hGcEyvhzmAwGL1hJdwHR60CulrnDahzznHnwIEDttZWYvR6j2mOHSvPVPZLEvi9e49kRQvoH3zwQVZZWVlePBb/RkMpRUdHB+/z+TQ5OcccwLHT73F2NhzBYIivq6vTT5kyxR3vsRNAr89nZmaGe+bMznJgS4/9HGekGo1mqPe+yepPnowxUsmBPhBBAA4ARgD/BGBHd8yUiEXIyeiBzmImBoMxKExAZzAYjMTBSrgPjloTYcyNzRgR1iXWVno+beeM3SUSTXNNXfopehdv5nsEgRs2bCh69dUPLti791DxRRddtHGI5fMGgwDAr371q0M//KHzi76eYDKZhhOU9vcdScb14vTYy5cvb1y+fPlrcTx2skq4J5JkCdtJcWuj53xCkEsRngDwHgArgAmQE0PnA7gYciKoEt3JoaEkOVkJdwaDwYjCYrFQp9M5aNwUdompMW4A1CskqzFmUuOcE8bkyZNd114brOrrMZvNFoz8XZIk/PnPj5138qQzb968eS3Lli2Lua95rOTm5orPPvvwB8FgsNd3med5ajQa47nIMtFxR49jP/bYI7vieOxkLQpOtPs4nQT06HE6AHwW3njEfxFyMkq4c4QQFjMxGIwBYQI6g8FgJI6YS7hDnS5sQKW9wMOobd5JSQaJHaIg2IUeJb3FLlHgTXxoNPYxHy6EIyDG3uXGosVzSZLwzDMvzPN6x+orKuqKX3755bHx6K1XUVFhLC0t9Si/cxwHu90e71LqZ4KkCvQJIJkO9EQzGpzuDgD7whsHoBDdiaFrw89phFwOvgLAKQyc6ElWCXcmoDMYjFQjFnFc1TFT+F+1xQ8Kap13wpB8EhdyhARNrqZH/+JAbUCvLdL6ztR5xZusrKxgrGXa169fX1RZ6S30+Yp0a9a8OGvZsmU7Rzq+y+XifT4fl52dHUT4c6zX6yW9Xj/SQyucqfg2Wd9J1gN99I3TV6wz1EXIlZDd6gOR8BLu4apdrIQ7g8EYELUGHwwGg5EMYroRI4So1YUNqNuBrsZ5J3TOwdagpvmJ5ild73XlKPvEDlFofaZ1QscbHQWJHHu0smHDhqLy8rZii+XXLlGcyz/99PPzJWlkMWJZWZn9hhtu/cajjz46OU6nGclgn5FkJAfSQXxOdZF+NI4jAagF8D7ksoUPAdgIoAVyD8BbAfwP5N6A5wCwDTBWQhM1TEBnMBgpSKwCeqztXtINVQro4TgZUNm8wyR0znW/q5td86OaRYH6gE7Z1/hw45TqO6svCdQG4qbupgqSJOGllzbODoXO5TMy7nAcONA0/r333ssZ/JUDc//9vzv77rv/d1EgEEjE+0kQ43UzQbCYaXDSUUCPZRxlEfKrAB4E8Cxkp3om5EXIPwGwCsBSACXoW59KRgl3DixmYjAYg8AEdAaDwUgcsd7sqb0coRrnrsbMX8LnrMnWBA2zDW2u3a6Crve6csQOUWhd0zqBipSzLLK0Jnr80YbiPg+F5vFa7flBg2GVt6Kirfjll18eO5Ljrl79zKymJnvWa69tm+P1epNVZjtZJKNHeaqL26NN2B4pw/0MuwEcBPA6ZDH9nwA+BmAGcBWAHwK4E8BlAEohV/5KZgl35qZgMBipRCzXLOU5qosdVCwkp8v95VBIypyzvplVKbklbe29tQsD9QFd48ONUzq3dk61LrFWngEH+hn/XCvuc6v1ZpfJtMjn803XrVnz4qyRHHPPnj0Zn3/eNL6yUirYuHFjYXh3unymkxEzAcyBPtrGGY4rvK9FyK8DaIa86Pg76HsRcsLjGeZAZzAYscAEdAaDwUgcrIT74KhVQAfUO++EYr/a3mica2x27XYVNP2taToVKZe9MrtSk9+zPOFIoRIFFXu3H6AiJYSMjrdWcZ8bDKu8hHDQas8PjtSFXlZWZv/00xOTTaZvehoauOyXX355TJxPu182bdo05he/+MUEmljrWbKSD4kimR++dBHQ4zEOhdzX7yPIDosHITsuTgGYCeAWAD8DsCz8fOsIxhoUSZKYm4LBYKQascQEynVajXGTWgV0AIBKW34ldM7GWUZn4e8Kd0tuSVv1naorOrd2Ts24IuNo/o/zjyVqTKfTydfX1+sGf2ZyiXSfGwyzAoRwsFhuco/Uhb5hw78mud0T9JJ0Ltm0acfkUCght2a97l8DgQC54447Fh8/fjwR4w04doqRLi53ILWEejeALyFX8noIwGoAu9F7EbINgAUJbD/MHOgMBiMW1Bh4MBgMRrKI1VmuZhFZrXNX67yTMudIt7luvM4Rb/EcAFqebilterxpUqSI3rmlM7fut3WzQ64QH+/xhsMzz7wwz+ez6wOBvVqn83GTy/WkiVIbGYkLffXqZ2Z5PKX6jIzvOAm5nL722tZxgUDc/7y9EEWRPPjgX5e99tq7xR988EEq378yB/roHSee7gMfgMMA3gTwMIAnIbsulGvDtwHcDeBrACYD0MRxbFbCncFgpCIhxC6gq/EeWilfr7a5q/E9T5ooaZxldOom6VqU3zO/mVmdyPEeeeSxab///aNn+3y+UXUvv2HDhqLKSm8hpXq0t6+xtrevsQYCFVqnM8M8XBf6nj17MvbubRmXkfEfrpyca7oqK6X8L774ImFCYCSrV6+evHPnl3P/9rfHIUlSoj5PzIE+esZI5XEogAYA/0bvRcgGyHHSzwDcBGAB5BLwcUOSJEIIYTETg8EYkFF108JgMBhpBkVsQikr4a4+Un219nBIypyVsu2ckQvqSnWd3q+8WZE90eOFfoq+y3fMl6WI6J1bOnM73+ycqB2rdfJmflQEYRqNRiws9DVkZa3tyMpa05GVtaYjL6+sJTtb3+73+4cs8ivuc6PxJi8hAmy2m501NSHrzp07433qyrXzNE8//fTE+vqOgmBwHPfEE/8URtrHfQCSIXCnuridrgJ6IsdphlzefXv4900AKgBMAnAj5MTQSgAXAMgd6WBMQGcwGClIrDETYnheOqJGIRlQZ8yUNBofbpziOeAZazzLeIo38n6lnHsixtq7d691377OwqoqIXfLli15iRhjuIiiSAoK+LbCwt3NOTkvtcvb+vbx472nBEEYVtChuM9ttgUeg6EkKEnnkW3b/q0VRTHep9+DQCBANmzYdG4gMEFbVvYV3n//fXsCh0v1eCOdBPThlFYfDokueR65CLkVcrus9yG70C8H8AP0XISsHclg4RLuLGZiMBgDkpTVbwwGg6FSeolA/T2PUqrWBU1qFtDVOO+EInZ19zxXyrZ3vNURcO12FYADtS21xa0PunWxtQ3A8faX2yef/MHJbAAwnm1szL09tyriaWf0Pd669c23+nvM5/Nx991335zbbrvtWGlpqTeW48nu8xJDdvZFbZLkITyfJQGL8eqr7+Dyyy8nFoslXqfe4+8miiJ54YWXL6B0MYzGu4Nffnmjbs2aNRNuu+22yngNmIakekIrHccBuhcvVwLYH/45E8DE8LYEcpl3J2SBvQJAFeRkUsywEu4MBiMFiSUhr7iwudHSLieJqF1AV9u8gQTPufFvjZMjy7Z7vvRY6n5Vt7D23tqFxY8U79LkxreC1+bN24pdrvF6jWZMaNu2snHLly9v0uv1o6L38MqVK2tWrlxZ09/j77zzTl57e7vuW9/6Vr/PiURxn1ut/+mWJD8BgKysyx1HjjzK796923zFFVfE69R7sXr16slNTd5ci+UvrkDgAd0//7l2/JIlS8BxcU83pct3Ml0E9HQbB5DjJg/kRcgfQxbLx0GOmSYBmA853qlBd9zUPJQBWAl3BoMRC0xAZzAYjMQR6ZIY6CZTzWKqmueuxnkndM6cgQtpCjRu6xJrs1K23X61vZHwhGpyNf54j2ddbG1zf+Lu9J/w2wEg579yTsR7jETx2GOPTX7hhbcX19XVW9euXfPRYM9vbGzU7t9/fBzHmULt7VdlKPs1GgFNTTq88847+TfccEM8BO1enxHFfa7R/Cio0SwQvN7zsXbtuoW33nprZYKSQansdGAO9NE7Tn9jtQP4NLwJAErQLaifHX5uLboTQ42DnWu4xC9LBjEYjFRi0BLuhBA1i6lqnjugrnknRZzSleicGVdmHMm/J/840N0TvW1D2wTeysfVJq24z7Ozr3MaDPnBkyc/z96yZUveihUrGjDKqwy4XC7+97//6xKPJ6i/8MILN5SUlAy6qHHHjh2FwSDP+3xPa93upw3K/tzcDOzZ85kljgJ6j++F4j6n9DIIwoyQXr8KX355e/bGjRvHfuMb3zgVr0EjSPV4IxnXlWRdu9JRQI8eKwDgeHgD4rAImTnQGQxGLDABncFgMBIHK+E+OGoV0Ed1oiBBJHzOnJajWTdm9UoOZHwtoykR43Vu6cz1n/DbOQMXlLyS0PR406S8u/LKiUBG9fvr8/m4l17auMDjyTN++OG+2V9++eW+WbNmOQd6TW5ubuC3v7333fb29h5lHU1GU0lWdtaSefPmNUbupxIF4Yb91T7994t0n/P8uSGACnr9PfTUqa8XpLALPV0SQckSthONsgojmQJ6f44rEbI7vRLANgA2dCeGFgG4BLITQ0kMVYZ/7wFzoDMYjBSElXAfGLUK6KP6njqVsV9rb4Dce/g0xllGp3GWcX8/Lxk2ivu8oGCmAwAImUcVF3q8x4o3//jHPyY1NnLZkpRBHn/88RkPPvjgF4O9ZuXKlZWzZu1ri96flZV1U05OTktXV5eg0+mkODnwT39HFPe5Xv89HwAYDAvR3n4W9+yzLyxYsWLFqTgvPE7WtSiVFzYna4xkjpOsUvHA4HOKXoRcjN6LkE+hO27qtWAnFAqxmInBYAwKE9AZDAYjcUiIsYQ7uhP4qoIQQsNOObWh1oUDaTNnpee5Urbd8aEjq/3l9slNjzch76688kSMGXKF+K53u/Ls/2FvUER6KlF0vtlZYFliaRFsQkxukcfuf2xOY6Mvx2pd4/B4vm/4y18ePvvxHz623zDd4OS0XJ/XLI7jcPXVV/cUyUVK6h6qm661a2EymU4ngFrWtIzzV/lthb8pPDBMEf30i9asWTOhvr6jQJI4zuf7lV4UCc9xHBHFDM1zz62/IAECerIc4ok+fqo7QpRxzpQrPFEMVazvAvBFeOMAFKE7MTQ7/Jx6dCeG6gBITEBnMBgpSCzXLOXaqca4Se0CutrmjXSJkRX3udV6qdfvb+MBwG4/y11d/WnOli1b8hJZzhwATp48qX/xxfWlK1feXFVUVDSkljgul4t//fV35wBXUEptZOvW52ffddfJrwZzoRcVFfmKiooao/e7XK4QgODtt//woowMk+eJJ/62Z4jT6QsCyIuO5d7nORpgI/X7N0KSNAiF9KSionncpk2bClesWFEXh/EiSeWYSRkjneKZdOiBHgk3hLFEyI7zKgDvQl6EPAFyqfeFkB3qvRYhMwc6g8GIBSagMxgMRuKI1YGuVjEVUPfc1TbvtHKQiC2iPrLnudIT3bXLlU8D8Ut4BRuDWiFbCBKBUF+Fz+T62DUm2Bg02L9ur9XkagKtz7eWeA96c4UcwWdZaOkY7Hgdn3VYX3t18wUkdAWv0ZwtajS3ih/tfPDsvfxeacZVMyqDp4LmnNtyKjkj1y2Ir20psS6xNulKdD2TRRwo0RPJ8ZoDpIGMzbklp7xlTcu49lfaZ1kWWap6DT4MdDqdOGZMVgOlBwlwEGazyWY0Gq3NzfypzMyMrniMkWSSldxI9YSWMs5ocIUnYqzhzEsCcDK87QBghpwYmgi5B+BFd9xxR8jtdrsnT56MlpYWezxOmMFgMJJELBW5lB7oaruHVnv5ekBd806rmOnzzz/PppQQv38j39S00ars12gg7tv3Zc4VV1wBSuM75R07dmTb7fbAOeec43j77S2Fe/aI47Kzt/hWrboDTKcKAAAgAElEQVR9SPGJ4j7Xalf43W5ibGvbmBHtQm9sbNTed9+vFqxa9d+HFixY0DnYMffs2WM6dsxrEwRXaNeuXccXLVrUPpw59kV2dkanILhF4A0QQkhh4ZiJnZ2d7V6vvVMQhHjf56bDouBkHD8dBfTR4kAfiC4Ae8Nbr0XIH3/8Me6///7g5MmTu7KysuoJIQKlNK6tKxgMRvrABHQGg8FIHLEmOlgJd5URdt6f6dNgjIDsldk10fusi61tYSEdcI18DCkgkfIV5d/Qjdc1lT5f+r5pjskR6gpV1fyo5qqW1S1c7g9yP/B95cuxXWk7EYt4DgDPfPjMmFqPy2Dmbg/xgZDGqPlmwOVYa3uu5vmsRwoePtT8RPO53sPe7OK/FH/CGTmp6e9Nkzo2dsySXNLBvO/nVUQei3AE2Xdk17a92nZe+4b2Gc6dzoJgY9BmWWSpKvhFwVcjKOEOhK+fK1eurF65cmV1xP6LACwAsHYkBx+AZCQFUt0dzhzoIx8rHklMF4AD4Y0AGFNUVLRk165dRQcPHtQCeIUQ8iWAreHt35TSYBzGZTAYjETASrgPjFoFdDUHTGnxXt9www01c+dW9CpnDgCFhYVeAHnxHK+5uVl7550/+YbVanatX7/29bKyuiJJmkp27z5S9LWv1dbH6kKPdJ8HAlatJHG8JF1FtmxZ38OF/uyzz04qK3NMJWQNFixYsGugY0qShDfe2JotinOlQKBTWLfulSmLFi0qG+GUKQAIgkDfeOPVNyP2CwDuA/AxgIMjHGPAsRNMqi8KTjdhO5kO9HjNqdci5GAwOLeoqOjsDz74ILuzszMbQCsh5D2E4yZKabyrNTAYjBRGjaWvGAwGI1mwEu6Do0oBHeqdd8rNOVAf0Hm+9Fgi91GREudHzsxEj81pOWr/un2f82PnjKqVVUskn8Q1Pdo0X2wV7dpibbPvK1+Obbmt2rbU1hrL8ZTe5wGphPORvdTreckYcG80hzCZ7qj4tKjaVB3K/1F+ma/Cl1fzk5pzGx9unNKxsWOW9VLrsdw7cyv6OibhCCzXWkA4QoONQRsRSGiE4nnKfUYikbwSJ7klPnIflSjENlET/jVZbo1kjMEE9NEzFgVQd9999724ZcuW/8vIyDgF4NeQS7/fAmA7gIw4j8lgMBjxhJVwHxhVuu+Z8z412blzZ/Zjjz02EQBsNps4d+7crr62/Pz8QLzHfuihh2Z1dnK2U6e6Ch544I9zOzryjZMm3dzc3p5vfuedLWNiPc4//vGPSQ0NyA6FdJzHs80A7KGhkIZrbdVkPvbYYzMA2X2+Zctn00VxirBvX+P4jz76aMD4cP/+/cKhQ13mzMxvODIyVrj27m0ev2vXrpHElGfyexG3sWtra/VHjx41J+r4A8BKuI/ecZSxEiHWuxYvXvzhmjVr/vqtb31ro1ar/RjAwwAKAawG8IMEjMlgMFIYNQYeDAaDkSxYCffBUfPc1TbvlHSQ1P2/unk1P6y51HNAFtGpSEn1XdUX1P6idlmgNqBP9PgF9xYczbsr7z3nx86ZB6ce/L5nr2eybbntM/1EfQsA+Kv8FirGlkxta2vThEJB3mqtdGk0vw1quQdCWu6BkMW426vVcsHq6mpTxtcymvJ/lF/mO+Yr6NzcOcOy2FJe8POBBXFvmRdUks+BipRve6GtJNb5UUohSZJeFMXcQCAw0e/3z5EkaTzkMmt8fy+L9fjDYERJAefbzjFdr3YVKyI6lShc77gKHK84xkk+iRvp8WMgmWJwOiWckKSxhtoDfViES5y8Rym9FcAYANMopS0jOSYhZDEhROpjCxFCFsTlxBkMhpqJuYR7DM9LR9Q8d0Bd807JmEkhEAiQ//u/vy166qmNF3322We2ZI7d3NysfeON9+YTchMNhS7kduz45OyMjPM9PK+jGRkL3bt31xTV1tbGFL+1t7cbLBafG3iW8PzqkEbzmKjVrhMFoSPU2NhiBmT3eVtbRkZh4S+bPZ5J+uefXz+9v+OJokg2b94uBINnc0bj1IDFcp7X45msW7fulSmxzo9SyodCoYxgMFjs9/tniKI4FcA4ANZBXpooRvxZFUWRbNq0vfi11z4qdTgckVVyk1VVKx1iJiA9BXQu0WNJkkQ4jmunlP6WUnoBgBzIYvqwYTETg5F+sBLuDAaDkThiKjNICGEl3NVHSidG1EThrwq/OLHqxKU1P665tOjPRTuan2qe5TngKcn7ft4ubZE2phKAIyXvnrxjTY83XRb+lXIGLmi70naC03Ohjo0dE1qeaUHObTkniEAG/FwVFhb6Dxz4dI3zI2dm+8b2CYZphjZNvsbj2OkoNp9vrs+6OqsRAPyV/tNJmGBj0Ex9lCNG0ufq785NnTmOHQ6YF5mrx/xszP76P9TPbNvQNhsAslZm1UiSZJYkyUoptYT/VTYLpdQKOeETfT8qApgE4DIA5QCOhf/1YZRfL0wXmpodGx0lXa92FVu/Ya3xfOjJDVQFbKbFpnpOf7qvfKonatLNgZ4UUTtMUvqtS5LEIezmDIvpx+J4+J8D+CBq36E4Hp/BYKiTEGKLmTDY89IUtQroao6ZUvK9fuGFF0qqq/35wWC+ZvXqp2fOnz9/d7LGfuihh2Z1dXFWg+F2j99fqXM6P7Q6HCcMeXnnu3Jy5rrKy3fnvfPOljGx9EJ/4IEH9p1zzmtNzz9fcVZx8Y/atVqrFAr5SWXlIzmXXJJxUnGfa7U3BwQhQ7Jar3Pv2/fA+I8++ujwRRdd1Kuv+Ztvvllw8GC7kJv7XYffL1/KZBf6n8bv2rXr2AUXXOCWJMkqSZIlOl5SfgZgijqsCHnB8RwADZDv944CaELi79fj8vk8ePCgtbISmYGAWfjkk08yly1b1hz1lFRurQWkn7CdcFE7goTPKRQKEUrp6Qo4lNJe390RwGImBiNNYAI6g8FgJI6hONDVWhFEzQK6GuedcnPWFml9458av/3E7SeWVd9RfTUA5N2d9+/sb/XugZ4IpIBEKm+svIwQImnHaet9R3wTfMd9edbF1s/Dgnllx8aOCY4PHY5YSrl7vvRYFPE857s5VYQjAAEcOxzFvJUP+Cv8NqVsu3GOsbXxkcbza35Sc67SE105DqVUCAVCVs9+T7HxQiMyrs2w+vy+5Rk/yrCQQpLnbnTP0Dl1WsKRyPc8BMBJCHEQQhwcx9UTQpwcxym/OwkhHq/XO49S2gl5BfhUAF9Hd+8yEYn9HI0oUBfyhIB1hfWkY6OjpPPZzkkAYLrYVK8/S++IOH4iYT3QhzcOkJx+fkn521FKTwvoCaCCUvppgo7NYDDUS6wxk3KNUxuqFtBVVro+ZRcNBAIBsmHDm3Mk6ULOZFrgKSv705TPPvvs0Pz587sSPXa3+/xmynF5oFTPUXoh2bfvz1Mo9YEQDsEg4cvKaoquuabx1GDl410uF79z58ESn69Q29T08enFxaKYwX/4YWVxZeVj1ra2jIy8vGvaAMBiudBTVzcp5/nn10+/6KKLTvdCp5QSSZJMu3aVzRRFQtra/mgVxdBEjuMEnud4my2bP3LkyPfmzJkTnZ/3RMRMdcrPSuzEcZwjGAyWBINBK+RFxlMBnA/gYgCdkBcfA4m9ZozosyqKIvn44wN5kjSRM5my/J98sjv/3HMd7VarNdHxnkI6lXBHksZJhxLupwmFQhyAuLeSCMNiJgYjTWACOoPBYCQORUCP5XlqTAQB6hWSAfXNO2WTQZoCjV+wC26xTbQCgHGWsSMZ4yriuWevZ1L+T/O35t2VV1F7X+2ctnVtF1atrFpS+nzp+5aFlg5NruawboLOE8sxDdMMrowrMk5al1mblbLs9qvtjbyFD1KRkvaN7bNsl9uqcn6U00pBLbnG3M7WJ1rPanq1aYL1P60UgOKAMABA1k+zAAGQqHQBkYgDBA7b9baTCMEl6IQuRRgPi+QeQgb+2FNKeUIIKKWtAI4A+AiyS30y5MTQRMjfnVXodlk0DuPPmzD4HD7AmblgyB/iAUA7UeuOeDgdSrgnU0BPG1E7TFLc7pEOdAaDwUgRYrneq1VEZr3A1TvvlEJxn9tsK11abWmwuXlDVrJc6HLvc8mm1Z4dCAY/4wnxS1rteQGf70NdXt77zvnz59cDAgRhvGQwGAa93vj9fi431+QymVr9QM81yhzH0x07vpju843XtrQ8kcvz4DmOcDwvaCsrXfMOHTpUPH78eEM4ZrIA4Fat+i7q6+sBgJMkSZQkySVJkkeSJPfEiRNbNRpNmyKMhxcVD3ofF46rgpBdrYcgu9HHAZgCOW4CgKsgx09HAVQgfmLhiL+Tivs8L2+eU6eziCdOHMyMcKEnS0BPxhjJiJmA5MVNaVXCHSxmYjAYg8AEdAaDwUgcsZYZlCil/fX6TXeoyhwFCmpdOJByc1Z6nvuqfPlZN2d91rWta3LNj2suLX64eLvxLKMzkWMTgVDeynsV8RwAiv5QtF/IEHz+Kn8mODmg1E/SxySeU0o5iUhm4yVGTVAMTlNKq1NKrfx83kopteQ+lGvVTtLO8gf8AADtPC2yf5nt1hZpQ2HXw8kIN4RT5MXcUCh0hdFofITjOH+cph4dKDsAfB7elgKYC6AFwHkAFgPogiymH4PsUh9JEDyipIDS8zzUHtLrZ+vb/Mf8GV2vdhXbrrPVcCYuGcF5MpNN6eLYSKaAnpQ5hf9fTdTn7XFCyMsAPADKAPyOUpq08qwMBiNtGbSEO2KPrdIRtQroap03kGJzjnSf63RTggBgMHzTmywX+hdf7C8WBDEoSXdzADhCAJ5HSBA46vP5NKtWrRq0bLtCKBSC3W7nfvnLnzRGxkvK5vV6LcXFGzJ9Pr828naL0swQIVleo9GoJYS0cRx3QnGMT5gwwVFYWHgjx3H7jEbj+3GceuR9ZQhAZXjbDuAXkEXzXACzwo+fQHfcNNJYdtj3tD3d5zlBADAaZwciXOgjPLWYSKa4nYwxkhXLJEOoV8ZKdAn3RC46ZjETg5EmMAGdwWAwEgcr4T44ahWSU9aNrTZqflozX+l5nv2t7JrMr2fWKD3RS9eWbklkH3TCEZSuKf0oen/BvQVHo/dRSjWhUMhCKbUqiR7PPk9e4FQgw/I1Cx92QJjFNpG4d7ph/boVhCchAJHlAOtM002K60FxQLjMs839BsmSJBlDIbnVcrymPcjjIchOi9chXzdLIDsspgBYAMCPnn3TYxb1qUQBctrN0b1fpGSw/vIKrq2ufKXnuf4svUM3Xdel9ES3fdN2ktNzzIE+tHGYgD4MElTCvQvAXyH38muD7Ga6F8AHhJCvUUrfi/N4DAZDXcRcwh3qjJvUKiSrMWZKyTkr7nNCLLSz8yWzvFeE2202JcOF/sor6zdXVlZG9wgHAEyfPv20UKyUVFfipfC/p/uMb9u2LevTT/eaf/GLezme7+FxcBNCnIQQh9FoPHXbbbcdVn5XXOMxLCamEdUk4gHB4NeEw5Dd6RmQ46UpAL4G4EoA9ZCd6ccARPcej2VsuN1uPhgMkoyMDHEoL1bc5xqNNdTScsQIADyvpXV1OlvYhZ4u5dWTKdKnTSwTMVaiS7gTyC3i4gmLmRiMNIMJ6AwGg5E4JMRewl1tyRAFNc9dbfNOyWSQ/Vp7pWm+qSH75uxaoLsnetuGtnGaQk3CxHMFSikopQZJkqx9JXoAWML/6qNe6vMd9fkdmxwCRDTarrWVhxpD/paft4ynQRrMuiTrPV2xrnOwkuoxkOzPceR4EmQXxQkAWwDko7tk4czw49XoTgw5MACuba58zVgNp5uho0p5e+/n3gz/cb/N9p+2GqIdPOGlKdG4NYUaj9LzXOmJ7j/it3J6TkLykg5MQB/aOEjSWAkvRQicdqDHNeFEKd0PYH/Ert2EkDcAfAngTwBYMojBYIyEWBb9qFVEBtQ9d0Cd806pObtcLm1mpuSg9O0e522xwOP1BrWDvHzEc83KygpmZma6JEmyhOOmSNe4xe12K7GTGT0X4UgAnIQQh9vtdj7zzL/Exsaga+7cd/dee+21VWFhPKaS6qOM6PvbTgCfhDc9gEmQY6ZFAC4B0IFuZ3oNBr+PJADov/71VpHT6deuWnVzuSAIMd/jdnR0aG02v4/SPaffe55HKDcXrtbWTh3SI2YCkuPYTlcBPRkl3DlKKYuZGAzGgDABncFgMBJHrIkOKYbnpCtqFdDVOu+Um7P1Ymsb5JXDp9EWaX19ucCHCqWUkyTJrAji0YkeRSBHz/s1CsClOB4IIdWR7gdCiIPneSchJGj6tglStTS7/fH2aeIRscOzzzMWEhxFDxe9ry/Ru/s+qzMPpcOOkxvD24cAbOh2WVwB2WnRgO6+6U3RL9YUajxivUgkt2QxLDDAt9eX4dnjydNN07XHIp4DgH6GvlcpRCFPCAh5Qmtfz08AyXSgJ5pkJWiS0pc8TCo70Psap4sQshnAKkKIjlIarzYODAZDfQwaDxFCJOD0NU5tqFVAV+u8U4577rnn+D333HM8EccOLyjW9yWMR/4MwBj10gAhxAG54lYrx3EnlHgpLIw7CCEexRX+6KOPTi8vD2VROj347LMvW6+55ppTPM/H1TEeh8XL0Qzn/HyQxbwvIfdNHw85ZpoBuUWWF3Ilr6OQy8H32Te9paWFHD7syw4EeGH//v22efPmdcZ6AkuXLm1ZvHhxn/ERx3EU8sLoRJNuPdDTZjFwmGSUcCeUUhYzMRiMAWECOoPBYCQOVsJ9cNQqJNMRiISpiuomLEmSKRgMju8v0QPAhJ6ffxFhB0Q4sXNK6ZsXkehxKQnkwSAcQeGvCw+KraLRsdMxFQDGPzN+s2GKYdSK5zEQayDdBeDT8KaHXDptKoDzAVwM2YUR2Tdd0s/WO4LZQerZ5dF5P/dOAQDdNF27eam55Qyc/0iOn2jS1YGejH5+SekbmCwBPXrYJI/HYDDSi1gWFKtZTKXA6QojqiHO5a5TArXNORwTk2AwOCayFVWUMG4FoIl6qTsiZqpVflZip/DWp/DbF62trZrNm3fP0mhuCJrNSzzV1Xflv/TSS8Xf+c53TsZxuskk1mtFCHKf9AoA7wAoQHd7rNnhx6vQHTe5lOMfPVpOvN4iDc+bpE8++Sp3zpw5XUNxoQ/y3HQqr55OAnqyS7gnvAc6pTTeJdwHQ1XXeAYjHWACOoPBYCQOVsJ9EMIJAjXOXa3zTnnCDgij4oAIuUIZVEfNIKdLqVulkGSBCIS0oUvC/cEBwBfhgGjkeb48IumjiOTeeLsS/FV+Y+BkIEv5vWtb1xjDFEN5XAdBQpJ98TyeD3Lvv0OQXRbjICeFpgE4N/x4OYCjmjEa8HY+KDbLcbRpiSme4rlCqpdXT1cBPW1cG8kS0AkhdgBXA9hHKY05Sc1gMBh9wEq4D0DEfZba5q7WeQNpMGdKqRBRUv20MF5dXZ1psVhMGRkZZsjiOPx+/3+HXxZC94JiJ8dxDX0I4654l1T/5z//OamlxWzPybmuXRAypc7OReTllzfPvummm2q0Wm08792SLZ4NZTwKuS96PYCdAOzobo91JYCrANQBONrQ0GA9ftzF5ebOdWs0plBlZXnmUF3ocT734cAE9OGNlTY90MOL0hIuoLOYicFIbZiAzmAwGIkjVgc6K+GuTlQ177AwPKrnHC6pblEc4304IJSS6rzYKkLySnBscEA7Ueu1Xmtthx9Of7m/wfOBp5Xq6LTsVdllOr3u83DfvGD0eCFniBfbRY1QIpzupU4lCt9xn8kwdeQucV+Fz1jzo5oloCDjnxm/ue2ltgntr7SfAwD5d+fHS0Q/E+/pSIL2EORShJXo7bKYFawNggboVM0YDcRWscG1zZVvvtzcqPREjwPJ+nulS8/AdBPQU7aEOyHkRQDVAL4A0A5gMoAfA8gFsDKeYzEYDFUSa8wEqLNyl5qFZEBd8x717sSokuqn4yVJkqwAIltRGSJfd/To0cCpU6c8r7++nc/Pz3D9+te/2A+gkFKaq9frX4pYUNzv3+Do0aOmioqK7KuuuqpXO6bhEuk+F4RMCQAyMm50VlfvSgUXeiK/Gx0A9oQ3A+R7vykALqqqqtH4/dnSlCljp7lc5FRT06TQcFzoA8BipthJdiyTrKpdQAqWcGcxE4ORfjABncFgMBJHrIkOVsJdfYz6xEgCOKNzliRJO5AwHi4PaI56mQjZMe4ghHQp5QE5jnO63nPZ/Mf8Go1d0+Fc5yyVKqQToCDOXc4JRCC+7P+XDV7g23ieb+/vnNrWtY3zVfoycr+Xe1hfqvdSiaJldcsE72Fv9phfjtmnydWMaHVy8xPNMyCBFD1c9L5hisFd+OvCgwDQ8XrHWRlXZZzSl+q9sRyHShTRAnJf++LEYAeN96AN4e193wFfcbAx+G3teC3VTdfND9YHiW+fr937mbfSMN/wOeFIcxzGS1YJ91QfQxknHQX0VC3hfhDADQDuhHytbAfwbwA3U0r3xnksBoOhPkJgJdwHQq1zV2PMdEahlBJJksxKvBT+t0crKsiu8eh8sosQ4gzHSjURPzsIIQ632+156qnXzzlypLawtdVvM5tbvUuX7vrqoosu4kOhkF0QhMbBzk2SJKxd+/Lk6mp35qRJk1xTpkyJS1uq559/vrStTbCFQp/wjY0HTlfuCgQMmo0bt0wfjoB+8uRJfUlJiS9qd7y/vwMdL973t14ABwAcqKiosO7f33J9RsaCPI3GXZKVhakLFkwP1NW1thw/ftwyffr0gwB6LR4fIsl0bid6jHQT0NMmPpMkKREl3FnMxGCkGUxAZzAYjMQRUwl3QghVWz+7CNQsoKtx3nEnqqS6NSyQR/YZVxI9uqiXepVygISQBp7nj0UmesKu8X5Lqufekss1Pdo0JVAfsJjmm6o7N3WeQ0OUE3KFjrw78z7TzNacPdi5Z96QebLxr42m5iebp+d+L/ew413HGM8hT459hb1ipOI5ABT+pnBvsCWoVYRypSe697C3Klbx3PGhI6v5yeazi/9cvEtbpPUBgP+kX1/7s9pFud/L3ac7L/rPmrqILaJPW6KVdFN1ZQAOaMZoJlGRzhQbxTmgmA+5b/pRyP3/ajB8IZT1QI99nLRJ0ESMlZIOdErpgwAejOcxGQwGI4JB/09VcRlzgAnoaps3kIA5h0uqW5VWVNHCeDhmMkeNHVlS3aGUVFeE8XArKichZMDv8NatW8fW1OiyTp2y5AlCbtDvt2tfeulf0xYtWhTz+e/atSvzyJFAvsuVq9+8ecvYKVOmHBvWHyKKc889t7mlpeWz3o+MR1FRkWOox1u3bl3R3//+7KLf/vZn2y+77LJEtIOKJOmLTI4ePWoKBkWPy7U/eOBAwwdGo85msRjyzGZdrtPpugrAFejZN324Cx3SIdZgAvrwxwESvPA4FAoRsJiJwWAMAhPQGQwGI3GwEu6Do2YhWY3zHtKcKaW84oDowzWuCOMWyH2tT78McpJHSfS0RAnjijg+opXGnJ6T8u7OO9bwYMOMYFPQRINUEPKFdk2OxpXxHxkNbs/geQLBLoj5P8w/3PhI44zGvzSeBQD2r9srbEttrSM5NwXewod4C99DKCccgXGm0RXrMYQswR/qDBmrv1+9ZNxj496nEsXJu08uoX6qEbIEf8RTk5m8SchY5kvNkQ5zH4AvtcXaL7XFWh7AeMglC2cAOA+yC6McsqBeCSDWBQ+J/t4zB/rQUSrAsB7oDAaDceaIJR6SgNPXOLWhZiEZUNe8KSAvEo75BfKCYkO0MK78jHBZdUSVVAfgJ4Q4IFfcauY4rjJKGHcQQjz9LSiOFafTyW/ffqDE4Sg1imIuLwgu0WSa5/3qq0NF+/fvd5x11lmDHkOSJLzzzvslgcA0ISdntmPPnpeKjh07dioeLvSFCxd2LFy4sGOkxwEAURTJSy+9MbuhIStnzZp1sy699NKdHNfjkpXyVRWWLFnSPHeuo1Wj0eR3dXUdinwsJyfHBGAi5BZZV0Hu+3wK3YuQY41z08kdnk4CelJiGSQpPpMkiZMkKeE90BkMRmrDBHQGg8FIHMpqSVbCvX/UKqCnfOA8DHrMOVxSvV9hPJzsMUUdI4huB0RnREn1yESPa6C+efFE8kqc5JN4z5eesQCgzdN2iu2ipeGPDTMsP7DEdAzexotChuAT20QDAOjG6+JSijBeGGcaXUUPFe2s/WntJVXfqVoOAERDQkUPFe00zjS6/H5/sr+/yRCgoz8/IQAV4S26b/rs8OORLouBFigkSxRmAvrQxgGS188vGeMwAZ3BYKQasVwb1Swiq3XuaoyZekAp5cILivtrRWVB/yXVlVZUJyMc5E5CiIPneQchZKRltmPinXfeKaip0WV1dJgsgjBblKQvuUCgjQ8GZ2o3bnw7a/bs2YMeQ3Gf5+Ze4jCZiv0VFR/mxdOFHi82bNhQVFXlH2M2f8t96NDz47Zv356TBBd6XyTsft1kMoVMJpMIQMrOzvZEPewB0AKgDIAR3X3TFwO4FEAb5HjpKGRhvb/zS4eYCUiv6mDKWGlTIUySpLg70BkMRvrBBHQGg8FIHLE60NUqIgPqnXvazptSSiJKqp9O9EiSlA9AcLvdd4UTPf2VVHdElFSPdkD4RuqAiBdihyg0PtI43bXHNZ4IJGS50HKY6EnIMN3Q7PzIOQHjgMwVmQMeQ+l57qvy2W3LbCfd+9w5Sjn3WEusJwPjTKMr/yf5ZXW/qbsEAAr/t3DHUFzssUJFSuofqJ9h/Z6VI4buRRDtT7RPNi42Nuhn6J3xHrO/UxnksfrwthOAHXJSaCqAKyE7LerQ7bLoK2GW6uJ2spI0yXI4pF3SKezOTIZQz2AwGPEilmujWkVkRNz/qm3uihs7LedNKdWEQqEewrgkSYUA4Pf7b/D5fAb0XVJdiZmcHMfVRQrj4WpbrsFKqieLSPe5x6Mz6vX5/lBIJJ2dm6w223jX3rNKwt8AACAASURBVL0f2A8fPkzOPffcfo8R6T63WMb7ASAzc7Erni70eCCKIlm/ftMsSbqAy85e2dnY+Hl2lAs97p/jzs5OftWqO5d/97u3frFs2bLmwV8RVwa7bnsA7A9vAoBSyDHTWQAuCD+uLECuQu++6emwWDfdHOhptcA5XMJ9VFwrGQzG6IUJ6AwGg5E4FAF9MFgJd5URToKl3LzDJdUt4a2XYzyipHpkRQUKwAlAC0DkOK4yWhiPR0n1ZNO6pnWC5JE0hlmGOtsSW63tCltT06NNU4JNQVPG1RlfSbw0Z9BjPN9aovQ8ty21tVoWW1oa/9o4vfnJ5ukF9xZ8GY8+6PHAf9Kvb3qsaZ7ye9OjTfMNkw3vKz3R44X7c7et442OOe5yN8l7MK+Ct/Bofbh1eteGrvPFNvFz/Qz9ASQ+aB/q97IDwJ7wZgAwCXJi6CIASwG0o9tlUTuM4w+VdBLQ08rhECbhiwIkSQLkOTE3BYPBSCVijZkAVrlLjaTUvKNKqivCeI94KfyvPuqlPshtgkAIaeU4riVKGHfGo6R6Mtm2bVteba0us7292ipJ1cTj2auX/z51nEazU8zJKaWHDx8WBhLQFfc5x5mlhob3rfJeirY2i2U0udAV97nNdqOTEA4Wy42uQ4d+lUgXOnnhhRfGHjzYOeHJJ58Rli5d+lZEufhE398O9UMoAjge3giAsehehHx2+PFKyDGT8pxEk05l4pGEcZSxklVNC0jwnBLRA53BYKQfTEBnMBiMxBFzCXeV9vID1JsIGnXlCCVJ0g0kjIdd432VVFccEB0cx53swwHhJoRQj8fzTQDEYDBsTf7s4k/mNzOraYByunG6007xvLvzjnmPes2mOSaHy+UC4Qb+aJvPN7fqxulc1outbUB3T3TH+47c0SSeKz3Px/1j3NsAUPvT2kuUnujIPf3UEX+mzeeZOwt/W7i97o91KxrubFikm64rd2x0LDAtNh3KuS/nwEiPPwSGOxcvgIPhTUB33/RZAM4PP64FkA1Ag94ui3iQLskmZZx0E9ATnnQKBALK/QRLBjEYjFRCwiDX4YgWPWqMHQAVxk3Jass0FCJKqvfoMx4ZP0FeUByZb6WQS6orraiqlZ/DVbacSkn1YDBY7Pf7b9Vqte8LghBrv+hRS25uru/ii6srzjvPd8Ltdmm7H8lHQcEc5/Lll0/Kz8+fMNAx3G63kJcX6pKkT3rkTOx2eLxezajIayvu81BoAa/VjhMlyUuMxnP8TU1TzIoLPfzUuH2mOzo6hI0bt5eEQrO58vK9JW+99daYa665pj5ex4+B4c6FQl5YXAtgO4AsdIvp14Qf7wLAA8iEvCA5ETAH+vDGSpv4LLzwmMVMDAZjQEbFjQaDwWCkKayE++Code5Jm3dUSfVeffPQ7YDQRr3UE10aMDLRExbJh1pSPWXf62BjUCtkC0EiyIk87RitX/JJXLA1qNFka4IAwOk5yTTH5Ij1mIYpBrdhiqFHyUHBLoiZX88cUeJDCkiEcATKuSqEXCGeN/NDChC9X3ltCIFTep4DQNFDRTtP/fzUYs+XHpthqSGu76nta7ZmUS8ebfhxQ6n/mN9uPM94NO9PeZ9ELUYYTQ70/hABlIe3twGMgZwUWgi5b/p09OybHq/Sk8yBPnSS4nAIk/A5BYNB5f1hySAGg5FKxBozIYbnpSs0XUuZD0BS3/NwSfV+hfFwzGSKOh8RgDOiwlZdhDCuVNwaNSXVk8m7776bk5ub6//5z396pL/neL3eklBo4FuW5cuXNy9fvjyh5cnr6up0Xq+XnzhxYnRP75j48MMPsxoaHFmEfE5bWq6xK/s5DlJFhWfM0aNHzcXFxfE7YQBr1qwZ29KiNdjtf2rr6rrX+txz6+deffXV9REu9EQSz+9kG4CPw5sJct/0CyAvOP4BgFZ0t8eqQ/zupZmAPnSS3WIrGSXcWczEYDAGhAnoDAaDkTgGdVJEPE9tyRAFtQroQBzmHVFSvZcwHuWAiIyiJchJHiXR0xQtjCeopPqoc5DEiuSTuOP/cfw63QRd44QXJuwkAqGST+Iqbqi4PNQRMk/9YOqrg7nNk0njQ43TQUAL7i04oojoLWtaxnkPebPH/m7sXs7IxRyIZnwto8l6iXULp+9+jXGm0TXxtYlbOD0ndX7WOYGfyvd4jXuv26qboPMINmFYnyF/ud+o/Cw2iTbJJQm8lVeOlYw/dLw/qxRysqcOwDwAX0AWzKdC7pl+NYBT6E4MxcNtxAT0oY2DJI2V8KRTMBhkDnQGg5GKxBI3sRLuLG4aFuGS6sqC4h49x8PxklJtq1dJdUKIA3LFrSae58ujW1ERQrxxLqmesjFTJI2Njdo77/zpf2ZkmJ179ny0ThCEUTsvSZLwpz/9fY7bHdA+/vgDu/V6/ZBFu4ULF7b//Oe37QgEAnz0YyaTKTh58mS3z+cjkiTh/vvvP+v666+vnDlzpmu459zc3Kx9/fXt4wXheonnsyWj8bue8vLvR7rQk3F/m4hjuwHsA2AGcB6ANyHHTOcAWBR+PLJv+khzFukQz6SjAz0pC5wlSWICOoPBGBQmoDMYDEbiiHXFPIVKE0HhsnxqTAQNOO9wkkc3kDAeTvYYo14ajEjytEeUVD9dHlApqZ7IyaUbnJ6T7Nfa9zf/o3lZ5S2VKF1T+n7FDRWXe/Z7JhXcW/DOaBLPAcC80NzY+lzr9IY/N0wruLfgSOsLrSXOj5zFtmW2E0MRzxUixfPIfe4v3Lbq66sX2W+3w/Q/JhCewFXmsld9p+p649nGyokbJm4f6liNDzVO7Xi1Y5xpqemg8QLjkZY/tFxe/9/1l43555h3I0T0RJKMHuUeAGXhzQjZZTEFwGIAl0J2YSh9009haImDZCZQ0iHhpIwDJKefX8JLuCsOdEqp6pxuDAYjpYnlek8BubpSgs9ltKLGuGnQeDpcUt3SlzAe0YbKCrkcdORxXREVtk5ECuOEEAfP805CSCLa7cRKSr/XDz744FldXcTqdHaaV69eXXrnnXdWnulz6o8dO3bkHD0ayBdFnnvrrbcKrrvuurqhHkOv10vXX3/9oK979913M1avXnfe55/vPfLOO5s3D++Mgb/97W/TW1s1Jrv9Jm8wCOh05wc8njnGJLrQkxEzUXSL5QRAEbpLvZ8DuR2W0je9HHKMNdQxkgET0Ic3DhI9FnOgMxiMWGACOoPBYCSOmEq4q1hEBlSYCAqXVNcA4AKBwJTonuPodkD0V1JdKQ14pA8HhD/ODoh4M6pPbiDG/O+YrwCg+cnmZQcnHZwFAlrwPwVv530/r+JMn1s01sXWNgCHW59rnX7iv09cCAC2ZbYTWTdl1UY+j0q0V5/2vvb1h2muqSvrtqxjbc+0zZGqpAty/ivnUNV3qq7n9Fyg8DeFZUM9b1eZy972QtsCywpLQ9ZPsw4QjrQD2Nbyh5bLW//YOifvgbzPkZygPZkLTDwA9oc3AUAp5KTQWZBLF3rQ02UxWCI3mT3Q001AT4ukU9iBzhJBDAYj1YhZQEcK30+OEDXGTQIASJKU4/f7z4oSxpWYyRz1MhHyYmInIaSL47jaSGE8vLDYPYpLqqf8QufGxkbtm29un89xt1BKK7hnn33x/Ntvv71qNLrQJUnCW2/tKA0GZ/EcZ6Zbt5aVXn311Q39udCrqqoMxcXFvuHMhVKKxx57qtjvz9Lt3Xtk2o4dO/YsXbp0yJWnnE4nv3nzh2f7fPlCW9ufjMGgpAGAUMjLl5e3l2zbti1/+fLlSnuoVHOgDzRWTXh7D0A25JhpCoBrIx5XFiF3xHBMVsJ9eGOlzQJn5kBnMBixwAR0BoPBSByshPvgpFUiiFIqKA6IaGE8wjVuRrjiQCAQ+Ca6S6oriZ6maGE8nPBJ9Rt7Sumoy5kMifwf5x9pfrJ5GQBwRs6Xe0fuqHVSWBdb27re6XIHm4NmAMi8PvNU5OOBuoCu7td187NuyToSFtwh+STu1C9PnWOYZmjNuS2nOpZxcu/NPUmyyJyW37Us6Hy7c4EmR9M+4ZUJr0T3do/sFR+5T8gUgopgbz7f3DH2T2O3aS7UFCpXBctySyNn5jbrz9J3De8vMTyCtUG941+O8Vk/zDrds5EGKGl9uHVm1j1ZX3GGoTv5wwyUdBABHA9vBMBYdLsszg4/rrgsjqNvlwXrgT68cZJFwku4i6JIRrEowGAwGP0x6HUr4tqmyspdSKO4KaqkupVSGhk7WQEoC4t1ABAKhRaG+2R7I2KmBp7nj0W3okpASXXGEHnwwQfPcjh4i8FwhzcUquFqa68rGK0udMV9npNzmUMQLKETJz7N7s+FXl5ebrz77l8uveKKC47dc889x4c61u7du7nDh09k6PVrfcHgr4S//OWR85YuXdqvC33dunVFV111VYPNZutRhYvneXreeTOOA3BT2sR1dnY2dz9WQvPz871DPbdh0OtLtn379tynn3527t///rcdOTk5gTgcf6B75lYAu8KbGXJFr6kAlgK4DEALuttj1fdzrHQR0JNS7jxMwqtphUnKnJgDncFgxAIT0BkMBiNxxORAh4pLuKO7FCNGc6IjnOTRDySMhx0Q0SXVA+GS6k5CSBvHcSc4jnOEQqGxkiRNNRqNfyeEeFRUUn30vsmDoPQ8BwfJMNVw0nvEO77ylspLlJ7oZ/r8omlZ0zIu2Bw0awu1jkB9wKKUc1fOVbALQc7C+Zv+1jQPwOfmc80dp+47dY7/hD/LttxWPZSxjBd1f+x1E3T1+kn6HuK58yNnZsUNFf+V/5P8zQU/LTgKAL4Kn/H4lcdXWi60HBr/9PhdynNtV9iavV5vYeRiC9OFpraoIRP19z4tDDvecJR0vtS5MNQeMuT+NncvDVBSf0f9Ev9X/nGGsw3N5svNTSMYJ1aXXW142w4gC91i+jURjyvu9DYAqK+v1xJCUFBQwAT00TdOUsYKJ4JG3TWJwWAwBoE50AcnJQR0SikvSZJ5oFZUACzoXVLdqYjhhJAW5edgMPh1QRB26HS6PYSQZLTzGS2M+ve6LxT3OSG3UI7LpxyXHwoELtQN4kI/I3ONdJ+bzRP8AMBx59GtW3f36UJ/8cV1E2tr7Tlbt36Bm29uPZGdnR1ziX9JkvDEE//UhELzodVeIlLaQvbu/WG/LvT169eP/f737/725s2by9avX78j8jGj0Sg99dRTuwOBwPRAICAAONjHkJZYz20EnH4vJUnC44//89wjR7omPPnkkw3333///ngefxBcAPaGNy2ACZDjpnkALgw/rjjTqwGIkiThyJEjZNq0aTTB5e6ZA3344yDRY4Ud6GzhMYPBGBAmoDMYDEbiGEoP9JQMkONA5N/ojCT8KaVEkiSTIowP0DdPE/VSd4TjoTbCDXHaAcFxnL+vMX0+n1aSpCkcx7n7epwxulDEc6Xned738yrqH6ifofREn7Buwo7R1Ae9ZU3LOKXnedZNWbWODx1ZkT3RiUAoZ+Skoj8UfVF7X+3cxocbFxBednXl3pX7hW2pLeZSgt6vvMb6u+oh5Ajtprmmyq6tXfOrb692j1s9bpfyNzEtMHXqJ+lP1v++/joAr9qvtdccv/L4ypAjZM68PvPYEKeXlD901g+yjoU6QgbnZuc8KlEi1os2/1f+cZl3Z+4YoXg+3PNvA/BxeDOhu2/6EgDLALRKknTs+utvmuP1evHRRzs5k8mUqNX06SagJ9wVHkEyeqBzaVCxhMFgqA8moA/OGY8ZJUnSDiSMh8VxU9TLlJLqDkJIZ0RJdQchRKm25eprQTGlFMFg8OuEELeKxPOUXgQn9z6XrFrtOYFg8BMeAHh+Yai29t+jzoWuuM9Npqk+l6tKCwBm82RvVVVZTrQLvby83PjRRxUTLZYbPE1NW+3r1q0bPxQX+nPPPTdu376jvE73fADgoNH8Z9DrfVTfnwv9kUceXRwKafj33vtgfnl5edmkSZP66+/d3+cl0ffrPa5Fb7311pjy8vaSUOhsfsuWXWd/73sth0foQh/utS4A4Eh44yD3TVdKvc8NP175zDPPuH7961/r//CHP+TdeuutIzjNQUmn9lpA8uIm1gOdwWCMGpiAzmAwGIlDKeE+2E2z2ku4AwkSSiJKqg/mgIj8+0vo7pvn4DiuoQ9hPB4l1dX2nqdsMogIhPIW3qeI50B3T3T/SX/GmT27PiCgkT3PlZ7o7k/dOZFP44ycVHh/4b6qb1ddRkXKWy+2Vg5FPPd86bHUfqd2AZ/HY/za8a8YphjcNffWuNvWtV188vsnQ+OeGFcGAJyek6a8O+Vfxy479o3639bfUP/benAmzl36XOlztstsLXGd+8joEajn3p+7HyEQ5xbnXADIuifrvYybM07GYYyRfhfcAPaFNw3CfdPfe++9eVVVx3UA8OKLL96yatWqjyH3TY93wjndBPRkLuBKSgl3MCcFg8FIPWK5brES7gmKHyilJKKkuiV6UTEAZVGxLuqlSkl1R0RJ9Uhh3EEI8Q230ljE69QWNwEpOucvvthfIgiiKEl3cYj4rvI8pN27d5eMJgH94MGDmRpNKCRJL/MuV3clOYMBgUOHDmdddx1OC+gvvrhuYmdngaWo6Jrm5mbRtnXrximxutAlScKTTz5zgc+XCZ3uFBHFdeGF+TPI3r1v9XKhr1+/fuzx4+WlHPdYSBR/obn//vvPj3ahjxIoIM9v7dqX5gaDczR2+x87OzpW2J588snpI3Shx+P+XAJwMrxtA5ADYKooilPWrn1xGgCsXv3sjFtuuaVeEISjADpHOF5fMAf68FCuHawHOoPBOOMwAZ3BYDASByvhPjjDcpJElFTvkdyRJOl0z7xwsscQ9VI/IcQJWSBvVUqqR/YbT1JJ9TPuIDlDpOSciUDohBcnvB+9XxHRRxs538npJfJaF1vblF7nCpJP4up/Xz8HAEBAnbuc44xzja3Rz+sP/SS923ypud5+t92mL9F7AKD4z8VfEA2RzPPNjZHP5fT/n70zj5OivPb+76net9k3BodVFgWCisQVEHdxuWrUxOyaGPWaxWjccjVG814TDW7RXHELBhWN0XtdQSSAsigCLlG2gRm2gWH2rZfprep5/6iqmZpmlp6erprprvP9fEqhq7tOne7qps75PeccQRq3aNzy7SdvPxYA8hbkfTqAeN5fNYUhYiePcharieWrf49WRgshJ2CGfOg0HEMlBqBSkqTKO++8Z5QgzCnlvBRPP/18xTXXXHO13W6PoXtu+m70Pjd9sGSjgG6U4Ky7T1SBThBEhjLgb6PJxVQgxZhRaanu628UFeQFxdpjqy3V1UXF1YnCuLKg2KiqcLN+5hnHG2+89lZVVVViFwIAwPTp0/1Gn09/3HjjjbsvvPDAwd72jR49Oqz+effu3e6PPqqa5PH8NCQIDl5UdGFHbe2/ipKtQo/H46ytrTXH5+OIxW7jjMU5ANhsiAkCl3bu3JmnFdAfffQv84BpnLFruCTVsJUrHxmoCr03DKtAV6vP3e77QxZLqSQIl0vLl7+cjir0dJ97I4DGRx55pHXv3t3fAp7A/v2/EN59991zLr300vMANKDn3PR0QAJ66nagty0S0AmCSAYS0AmCIPSDWrgPzBHvkdJS3asK470leiAneXprqa6K4TWaaghtomcoAVw6ydhq7CFgRp9HLFJYEtSZ56W/Kt3kO8XXUvNfNbPUmejJiOiCU5DK/1i+OxaLHaN9vOKBii8SnxuuCrt3X7b7+8zFOi0+S6DljZYznMc4G9WZ6COErkBdO/O88FeFK6PV0Xz/u/4TAaDk/pLP02AjrSxatGhyff3BUmAJgGLU1c0QHn744TV33XWXBLll4aWQv4MH0D0DsDVFc0b9e5WNFei6i/XxeJwxxqgCnSCITCPZ3y2zx009fJckydGfMK5UjSeKmTF0d9tqFQRhv7qYWNNxK2jAguJkGSnnYRQZ7W9RUVGsqKhIj0retON2u6WpU6cOOFLtpZdePrq1tcQ3evTsxni8QwAYrNbT4++/vzypKnS73c537vx6UTAYvMNisax3Op0btPu1M7jV6nPgFc6YBYLwSyke/5+RWoXOtdXnXu+pQQDwen8UbG3936FWoevyOx+NRtmSJUvnM3YB5/znjLH3cM8997VdeOGFq20222QA3wQwF4AfPeempyqykoCeuh3obYsEdIIgkoEEdIIgCJ3w+Xzc7/cn057dNC3clZbqOWqiRxTFSQDQ2dl5Befcq4jjXvR8P0RoKiCUluqJwnggwyruzJz8MxMj9jOON8dt8da4WzvzXJ2JHt4Vzkm2Cj0ZwlVhtzrzfOKLExf75vqaK8+t/JY6Ez1BRE/mPdM9aG96tGmaOvNcadu+HwD87/pPdM12HfZd7Duc4qHTnnSQJAnPPrt4PmNzOefzlffvCv7CCy/P+uUvf/kXj8ezHvLv6mTIMwDPAnAu5CoMbZVFsueVjRXoRgroerdwF0CJIIIgMo9kfxslznnWd+5KaKmuCuF2URSPCQaDY6F03AJgT3hpSDN66pCmglxbNZ5yS/Vhwqxxkxl9HnHE43G2adP2MXa7N9bY+Iseo7uCQcm5cuXK0quvvrrXKnYtgiCAMQZBEHoI5ok8+uhf5nE+CoIwhnMur0nm/AK2cuX/HVGFzjkfzmuEAcCKFSvK5NnnJZbW1tu63p9YLMe2bNm643/9a//XPp8vlftSXe6ZH3vssWktLQ0FwO8BAJzfh+bmkwoeffRRdvvtt/8f5E4cY9A9N/1EyHPTd0OOmXYDCPd27D7IRgHdiIW6qk96t3CnuIkgiAEhAZ0gCEJfkrnhy/gW7kpLdVdiBYTSUl1bAZHYUj2mvN7OGGsUBKE6ofpBbaluuE8GkJVODYAZfR6R2EfbI+OfGb+OWburiwS3II3585gt2scGQd+v4YDgFUJjnxj7htq2fcoHU97YtWDXpRAHfU3oeQ11JR8Kf1m43XWCq8F7jrdB3Vnyu5IvXSe66rzn92xPP9zI1eeHyoD/RneH+R8zv//1nAcffHDm/fff/zmAAIDPlc0OYCK6k0JzlP3aKov+WrKSgJ46NAOdIAiid5L9bcx4MVXTUr3HKCptBTmObKkuAWCcc5cgCM2MsfpEYdzgluqGMsxCoaGMoMp/oxjR/lqtVn7//beta2pqciTuEwQBp5122mAXHffpb0NDg72qavc4xgDOT+v6/jMGHo9z6+uvvz7urrvu2p6kHSPu13lpaWnn7NlH7ZCkDgZ0aHYJ8HonR6xW64j5fNXqc+BkDpQyOW4qBXAKX7Jk6Rk333zzNrvdLkGOhfYBeF95whTIgvrl6J6rri5Cbh/AbLYJ6EbZUa9/qkAnCGLYIQGdIAhCXwasLleC5BGbFOCcC5IkeVRhvLdED4AcHPlvSkCT2NmvrRpX2wTG4/Gjo9HolS6X6x+CIESGwb3hYsQEkgZiRp9HNL0J5SmI5wP+djknOUPTP5u+RPuY4BSkqaun/u8gbRmG4BIkrXiu4lvgG6p4nvYkynvvvTdD/tOPj9i3atXqGYqAriUKYIeyCQAq0F1lMUvZr52b3tmH6WwR0HUXtTUY0cKdZqATBJGJZHwLd2VBsaM/YVwRx90JL40xxjogt1Vv0bRU13bcCgaDwV9ZLJavXC7X6mFwbzihGIIYVmbPnj2QSJoWSkpKoitWLH+yrq4usegAVqtVOu+8846ITYYRBgAnnHBC++LFi/VoLZ/2OOCDDz4ok6vPGwCM62GrpQWFH3zwQdlFF12U2GWsXtnWQs55TVG28wBcAKAOspBeCaCvDmXZIqAbImorUAt3giBGDCSgEwRB6MuIbuHOObeJoujrSxhPoqW62h6whzCuVEAkW30PjNBEmI6M2OQfQaSCgV0i9BRVjQjU037+f/nLX5Z/9NFHXwBAYWHh1Ly8vBOqq6uXAsCJJ57YNMDL1SqK/QBWAChGt5h+mXKu+9Fdnd4GqkAf0bZEUaREEEEQmUjSLdwxDJ27OOdMs6D4iJnjkDtu+dB7S3VtzLQjcRQVYyyS5H2UWeMHs/ptRp+znQE/U0WsT4dgb0gFuo7HTvs984IFCw4//vjjf/P7/TYAmDhx4vfa2tq2NDc3V/p8vtiCBQsGGtHVAWCzsjkAHA05ZjoZwDxlvxoz7Yd8P55NFehGt4oH9G/hTnETQRADQgI6QRCEviQtInPOWbpatmlaqvcQxjUz9NQKCGfCSyOaCogGpaW6VhhPd0t1swroAPlMEP1B1UZJMnHixNDEiRP3KH8tBiDNmzdvT3+v6YdGZVsHuX2sOjf9bMiVFg0A1FaV2SSgG9Xy3JAW7iZs/0oQROYzbBXonHOr0lL9CGFcUzXuxZEt1dUFxX6lpXoPYVxZUJzOxLxZhWTAXH7Tv+HEYBiu6yXjvpOCIODKK6+sSXi4EUAqcVMEwDZlEwCMRfci5NnK/t2QF1XpvegrGwV0o1q4C6DRVwRBDAAJ6ARBEPqSTHW5VkQe8AZRaanu7asCQjM3r7eW6h3Ktl9TDeFnjHVYLJYOxlhs0B4ODVMK6CYVN8zms9n8TRfJ/BZkcgW63sdPp+jsB/CZstnRXWVxjLL/5+g5Nz3dq/eNqggwugJd9xbuoEoKgiAyj7TPQFcWFDv7E8aVhcWJLdWjyoJiP2OsWRCEvQnCuLqg2Oh7PbMK6HRPTWQT6byeGYb3NyFTYpr+bKQDCcBeZVsOoAzdc9NdAE4DUI7uuekdvR8mZQyp1jbQjtYWtXAnCGLYIQGdIAhCX5JJdKg3oExpqd6nMK6I456EY4qQK8bVqodDWmFcSfQEkmypbjSmFNBh3gSYGX0e8YS2hrzOo50hwSl0/UbEmmI2sUO0Oic4+5p/PVxkwzWkd7JJD6IAtitbNeT27jsgV6ifqOzfDTkptBtAOA02s7WFu67/FlMLd4IgMpRBtXBPaKne6ygqyPNqbQmvcVznSQAAIABJREFUDyoV4x2CINSof9bETH5BECJp9SxNKIJ9NtwHDRZT+s05N53PmcSSJUvGjB07Njhv3rzmgZ89LOgtQBpxffY492AwaHE4HJLVak2XT3rFAHXK9hGA3wCoBWABcD6ABZBnpauLkOvTYC8bK9CNauFOC48JghgQEtAJgiD0RQS6KiDc2goITaKnHACCweBvcGRL9bCmAqJeEITdCcJ4B2Os08D5w+mmq339cJ/IMGA2n6l6ZAQihSRh/8/3n28vt7eM/9v4DwWnIMWaYra91+w9FwCf9NakZUzo91Idcddxy7MtRzuOcbR6Tvd0JbSCa4LF0X1RX/41+f216DMk0cRFjvZX2sfZj7a3u092t6o7AysDJVKHZM/5Vs7BIdow6rv2AeQqi1J0V1lcju656mqVRapzHLNRQBcAxPU0EI/HhTS3DCYIgjCCriS5pqX6EcI4AFc8Hj81Ho+fgZ73INqW6h2CINT1Ioynu6W60ZhSSFYwk98UM41wdu/e7f7zn587u6DA07Fy5Slv2O32ZD+zEXsd/+IXN59WUlIUuOeeu/89iJcZtig4Ho+zJ5742zFlZXnBH//4B3vTaMeI79tBAGsh5/qOhhwznQLgDMhxkhoz7UdqgnE2CuiGtHBX8pCZfF9AEIQBkIBOEAShI/F4vCQSiVzBOXdDXnWqwiG3VPdDuWGzWCwbBUFoVWeOWywW/zC0VDcaM1egE8SwI7gFqfSm0k8OP3j4zL3X7j2j4sGKj/f9576z4k3xnIoHK1YMIJ73S+fOTk/0QNSde25uo/qYGBAtrW+1lhd9ryhx/lyy9Ct28ihngfcDx7S/3J5T8v9K3vec7mkOrgkWN/y+4TxrqbUl7wd5e5l12EYoMOW/nFmZFPggMBYA3Ce7WwMrAyWhj0PlrtmuujTYMTp5Uq9sayFX+6lz088DcAHkCgy1ymIw/hklbOs+l1yD7j7F43FKBBEEkYnwzs7Ok0RRvBByy1stakv1DgAiY6zearV+rXbcGsaW6kZjVgHdrH6b0eeM4Mknn5ze3p7vCwb97sWLF4+//vrrU5mh3Suvvfba6E2bNo1auHDhliEeKunrZ82aNcVr126fabMh9p3v7K6aNGlSMJ3HT5Ee98wbN24s2LWLlxw82BCZP39/3dixY9PVJc2IuEm1EQawVdksAMahezzWScr+3ZBjpmrIc9STtQHo74shorYCtXAnCGLEQAI6QRCEjihC+H4AB1VhXEn0dLVUj0ajx0aj0SsdDsfGkdoyUEdMLaBzzpHB3QNSwVTOZgoFVxYcArC69oHas3ddsuvbzMbiYxaOWe6b62vRPk8MiBaL19IjwJTCEuMC7zWwrf3v2tmhf4fGV/ypYlnegrx6MSBaqr9ffW6kKlLuPcn7ivNoZ6if00opWGZ2xssXla+o/Vnt+Q13N5yfc0XOlo5/dsy2llhbRz01auUA4rkhgToTGM+5KudAx2sdCHwQGNu5ubNEbBFdrtmuOt8C31AF9OGeF9gBYIuyOdA9N/1kAPOU/aqYvh/9JyyysQLdiBbugt42CIIgdEASBKGRMbZbM5ZKrR6Pqk8KBAI3C4Kw3+FwDFVcykTMKiRn+8KIRMzmb0axe/du96pVX85wOG6KxmI7rUuXvnnCNddcs7evKvQPP/ywcNasWW0+n0+95+3z8w2FQsLChf9zRmNjsGDevHcOXnzxxYYsrH3++SXHRSKT7ZFIq/2ppxbNeOSRhzcmcVwj7p85IC8OXbv289GSdCxrb290r179Udk11/xwqFXoRv2W9vU+iZBF8moAywCMgrwAeQqAGcr+feiuTvcPYAN92Eknw9HCXe8KdGrhThDEgAgDP4UgCIJIFYvF0uF0Oj9zOp2f2u32HTab7ZDFYulImEeu3hSa8TfZrAK6ipn8pmTQCMY339eg/llwClHPNz1t2v2dlZ2enWfs/G7DooaJ6mPx1rj1wDUHZrb9rcdTuxj72NgN9nJ7c82dNQta/691lCqel99TvmIA8bxXotVRtxSWeiQhxA7RGq2OurXPsxZbo+XPlL/Pw9zR/lL7aVzkwqhFoz6wFlqHu6NH1/edCQw5V+UcAAfEFtEl+IRoGsTzHjZ0JNmERgTANgD/C+AhAEsgJ4EmA/gBgNsAfAvAdMhie292jErQGCU4617tThXoBEFkKNzhcOxxOp3rHA7Hv+12+x6r1dqkFc/V58Fc989azOy7Wf0mRhhPPvnkdL+/2OPzXRn0+X4UqKmJlC1evHh8b8/duHFj3rXX3nT1Lbfcckoyx160aNGk2tpocTg8zvH008/PSvKU+r2vXL16deGHH35Y1Nf+NWvWFH/99aFJLtdPOu3270bXrv16xu7duz1J2taTru/8xo0bC6qqUFRWdmpHfv5Joc8+ayzfv39/YqeSVI9vZAV6fxwGsAbAIgCPA1gJOW5YAOAWANcBmAugpA8bSNLOUBiOFu56z0CnuIkgiAExo1hDEARhJBIGDvhNKyJrWi2azXez+k2MQNSZ54JTiOZfmv+5FJKce6/de4YUlrruEx3jHSH7OHv94YWHz2tY1DAx3hq3Vl1VdVF0T9TnOrH3/IW10Bqb+PLE5bYSW1vNXTUXh3eGK8rvKV9R+O3C/mZ89/qd4HHO6u+pn9v0p6ZRUliOo8UO0Vp3a93pDb9rmMPjvMfrIlsjuV1/kSBEdkRykngrDJmBrh4/uCrYlQCRApI9tDGUnyYbIzF5IgHYC3lm+mMAngbwCYAiyCL67ZBF9dmQ28CrdrKxAl1XW6Io0gx0giAykWTFYTOLyGb13ax+m9HnEY1afW63XxUTBA93OI6JS9IZbOnSN0+IRqNHfF6PPfb4rI4Oj3fVqk3HVVZWetDPZxoKhYRXX337ROBc5nbfHNq6tXbCO++8UzbAKfV3jTC/349HHnlh5sMPPzsnGAxaenuSXH0+xe5ynR32eK4MBQIlnqeeWjRjALuqbb1noHO1+pzzacztLo0VFEwPtrWNcq9e/dFA700yxwdGjoCupQ3Ap5AXHz8EeTFyG4DTANwI4FeQR2WNg6zrGC2gG7HwmCrQCYIYMZCAThAEoS/JCOjqDagZg2SzCslmrcY22+c84pFCkrD3mr3nqjPPR987+utRd4xaHd4VHr33p3vncUm+VAW7wCcunbjSfZy76vCfD5+/7cRt10droiXlfy7/2jnT2efxmYNJzM7i6t8tPktKVeDMynjB9QVbYgdi9uZHmnNitTFH3a11p8fr4vn5N+Rv0bZmV2ee2yps9aNfHP0Pa6m1peHuhvOD64OFqdjWAa6deV58T/GXjsmOlsAHgbFpEtEz4felDsBHkIX0xwCsUB4/H8CvAfwMQB56r0xPN1nVwl2ppKAW7gRBZBrJ/m5JMG8ey6xCcibc16QTs/kLZMh1/eSTT05va3PmMObkgcDbrkDgbZfFUib1VoW+cePGvI8/3jbd6fxVJBQqcC1cuPD4/o6tVp97PNcFnc6zIrHYsbZBVKH3ysqVK9HQ4PHW1lqLly5dekSVfHf1+bWdjAkQBA8fYVXoXK0+Ly09pQMABMGCgoKT01WFDhjzfRuKjTCArwH8E7KY/jKAKgDTAPwIwG8AqB0O9B7Tm3Ut3CVJotFXBEEMiFkDD4IgCKNIugJdWf1oNlTfMyJo1gEz+W3GZNCIR3ALkvcU776KBytWqDPPC64sODTqjlGrfaf7DjCh+xIV7AIf9z/jPlT/7pvj2+6c7gxy3nMGuugXLfH2uFWdeR49EC0ZdduoDxzjHHU1d9YsaFvWVtrX+YS+CvkSHwt/Fc7hcc488zxNhbcU1kV2RayHrjl0SawmVlx0e9FazxxPs/pcHuWs6aGmedYSa+uoRaM+cExyBMufKX/fkm/xNz3YNDexUj0BQyrQucgR2RXJU2eeq+3cHZMdLZEdkaEK6EZVoKfTRjuATQBeBPBnAG8AaIFciT4FwM2QhfXx0Cd2yaoKdGrhThBEhpKUOKx0rzLT/bMWTjETQQwfhw835uTkBP1W60NxQfi9JAi/lxyOJRGXKxI+ePBgjxjmsccenxWNVthdrh91Wiw/jq9atem4xsZGAOCrV68uqqqq6hpBpa0+t9kmi4xZ4HReF06iCp1Ho1H22GOPHdPc3GzT7mhsbLS98846OBwXxRg7i7/99oczEqvQX3xx6Yxg0O3u7HzT1dJyU15Ly0150eg6u9/v9D3//PPHDvB2GFGBji+++LpIFAWhtvZvuVVVfyyqqvpjUVvbMlc4bLVt2bJlKIujh3sGeiqIkMXz9wA8CuBZAFsAqPHjLwB8F8AsAN402dQyHAK6ruK28m8qxU0EQfSL3quTCIIgzE4yN3xmrcIGzOu7Wf02m78ZQflvy7clPlZwZcGhxMfirXFr9feqFzArEwW30NmxpuMb0f+MRn1X+Jj3ai+YwCD6RcvB/zp4IrMxMd4Sd6szzwu/XXgw/9L8Q9Xfq76g5s6aBc7Jzn8kzkFvW9ZWevCegxcW/LKgIedbchfv4JpgccN9Def6LvJ9WfSbom32ifaYKuozC5Ocxzl7DGBndsZLHihZaRtjC6kzz9WZ6PHDcae2Un0YYADALIwXXFewm9m7z6VrJnp8yN+RTBTQtYQBbFW2nyh/bwVwDICTlL/vhjxLvRrynPWhovtcciNtiaKYUa0IGWP5AGYCCEF+b0RlU/8saR5jADo45w3Dc7YEQegIR3K/j6YW0GFO383qtxl9HtG8+upL/0rmeWr1uc32hyhjDrhcPwwHAi94X3nlFeGHP/yh9aGHFp86Zkxe86JFj28QBAHPPPPM0bW1kRJJirGOjoe8AMA5RyTidT777OITLr744mV9mGKvv/562d//vnxCc3Oz6w9/+MPn6o6lS5eOP3jQivz8/wg5HP5Ibe2q4qVLl46/7rrrqtTnzJp13EGHY2scSLytGo2ZM2fWD/b9STMMAL/wwvNqTjjhcGNvTzj66KMDQzw+kLlxEwdQq2yHAVwF4EMARwO4EMBFAA4BqIQcN/X6Hg4SdSGzkTPQjahAp7iJIIh+IQGdIAhCX6iFe/+YVUimamwio+iaeV4TLan4U8VbuRfm1lV/t/qc0L9DM7mVA/swrfSXpTsP/tfBE2N1sZyyW8o2MxvjkT2RalWMV2eiN7/SPDZRPAeA3PNz69veb6tqebrlOLFdPGAfbz/ccF/DudYya0v+T/MrxQ7R2vxwcymzMcl3se+rwIrAsfW31s8pfbh0ncVn6Qp8Xce72gGASxxiq2izFlpj1mJr1FpsjQJAvDluU8X1BAxbVa8Vz7seExhgH7JtI35LjZxN3gFgmbKNAjAVclX6DMiJgX2Qk0KVAPxDsJM1LdxFUcy0SorTAbwFIAbA1s/zRAAWAG8DuNSA8yIIwlgkdAul/f0mUwt382G2uMls/mYMgpDcT89jjz0+Kxwucni9xwfj8WoLADB2vvTaa+9abDZbaX19ns/vD7pWrVpVfM455zQ6nU5xzBhvLbAp4fttQW5u7hExk0okEmGvvvrOuI6O0pyVKzfPvPHG2m3l5eWRuro6x/vvbz7a5foxrNYcbrfnxuUq9BUzvvvd7+71eDwiANx00027AOzSHrO+vt6+ffv23Pnz5w8kuBpy/zxhwoTQhAkT+nwPhkCmC+iJNgBgM4D1AFwAJkOOmeYAOBPygmQ1ZjqQ4jllXQv3DJyBTnETQQwDJKATBEHoy4ACutKKEDBnMsjsArqZ/KZkUAYjBSQrRAgVf6x4J/+y/MMAMHHpxJV7b9pbaqmwjA9uDlbs+cGeCmZjYtmtZZu9J3vbAMAzy9OuPY610Bor/XlpVW82mMBQ8UjFlgP3HihvXdw6GyJE2xhbffmi8pWCVxBrb6ydBwmOotuLOhxTHJXOGc7mpoVNc+p/U3/6qL+OWptYXd78aPOxka2RUSV/LFlvK7NFAKBtadtY/5v+aaV/LP3QPtGuRzKmP4aUCAhtCBUIeULMOc3ZJRTHDsackW2RXO95XrVKJNMr0Puzc1jZ1kCejz5F2RZArrSoRXeVxWBW2hs5M1z39y7TKtABbARwOeTuAlbI90LajWn+74bceYAgiOxDe2/c3++kWUVkwLy+m9VvM/qc8UiShM8++2qy1WqLh8OXONTHGQMPhx14881Vo73eW+o6O79y/POfb08+66yzGm+44YbqG264YdD3N//85z9La2oiOYWFC1s6Om7xPfXUU9P+8Ic/fP7CCy9Mbmiw51ZUONDautwVjXJmseSKtbVC8UsvvTT++uuv7zUWA4CFCx89/quvasaMHj162eTJk4Opvg9pIC33zIsXL54QjcaF66+/LtFnIwV0vUn0pRPAv5XNCnkM1lTIC5BPUfbvQndHr94WlidjR0+ohXvvUNxEEMMACegEQRD6kszNmJnngJtRSO6Cc84YM5XrpnI2m77T9gp7eMq/pvxv4kz08sfLd8Y6Y2PrfyLrt5Y8S6fnm562vo4zEExgzHeer8m/zO8BANdJrn2WPEscANynuA+4TnaVOqY4PADgmedpArAuujfq6601u+9C34HOzzrHNNzVcHrJH0vWB1cHyzr+0XG865uuPbbxtt7Ec0NmoKdyfC5xtC1pmyG2ip7ie4vXOac5/bGDMWf9XfVzIYG557lXCk5BTTBki4DeX7vzNgCfKpsTwCTIiaHTAMxX9murLPpLvhhZgW5EC/eMSgRxzhsBvDnc50EQxLCjVqAPhFnFVMDcvpvVbyLDEAQBr766ZGlNTY07cd+uXbt+8s47LVJ+/tyAyzUhum3bA0epVeiDtRMKhYRXXnl7PHAOHI7jYuHwleLKlc/NvPHG2m2NjS2e/PxQwO3+R0Eo1CJKUlS0WiEyxiNNTU1HnJfK559/nrt5c+3Ejo4C30svLZ1y//2//7yv58KYGehDOn5NTY1z6dJ/zZYksPPOO+fQuHHjOnt5WjbETf3FmHHI4692K88rR3dHr5nK/j2QY6ZKAP0tmjBE1FYwpIV7plWgU9xEEMMDCegEQRD6Qi3c+8esAroZq7HN6HNWoRXPASBaG3XU3Foz3T7Vzpidia5vuA4HtwSP2jF3x1UlN5R8UvT9oprB2uj4sKOw/g/1U+xj7QeFXKGl458dJwkuIV5wQ8Hu/Gvz9wD4BuTV1ABkEV0R0o/AMdURKPldyfqG+xtOP3z94QsAwPVN156iu4q+SvRlpMMEhuL7ijfW31k/p/G+xjkFNxZsbv1b6/E8xi0l95es1YjnRiZp9CZZX8IAvlY2C+QqiykApgE4GXKVhXZuejRFO+nAiBbuGZUI6g3GmAXy97wAgMg5PzjMp0QQhP6oM9AHjJuUhLcZMauAzjk3VRhhKmeRhf7OmjWrfdasWT26cNXU1DjfeecTZrcfL23aNG/6+PG318Rix1rVKvRk28OrPPPMM0fv3x/25eb+MCyKgNd7daCt7Z+5Tz311LSHH35oI2RR9EYAKwAkdR/1yiuvTQ2FJrh8vrMDGzY8PXnXrl2Vw1yFPiRefPHFya2txTmAgL//fcmUe+/93Zea3SMtnhmqDSRhh0Oei34IwCrI99lTIAvqFwG4GPK1oi5CToyxqYX7CITiJoLQH7MGHgRBEEaRTKLDtC3cNe3rTZUMMqvfRHYROxxzdG7pLGp7uU3IvzR/R/md5dtEv+gK7wxPbH2zdepgj+df6y849NtDZ9gqbKFRfx21btQTo9a5TnZVti1pO631b60TlKcN6jvjmOoIuI53dQn5eT/Jq+xHPE85UBfbRGvD7xpOEP2iRX2MSxyNDzR+I7Ir4hnq8QHAVmaLlP6pdB2XuND0cNMcsVX0ltxfstYx2aFNbGXbDPTB2hEBVAF4D8CjAJ4FsAVAGYCrANwO4LsAZgHwDsFOqhjRwp1BribJSBhjxQDuA7BX2V5UHj+ZMfYUY+zY4Tw/giB0I9l7Y7OKyIB5fTer32b0OWtZunTp+JoaC6urW+3q7Ox07t37SIXNNi6+bVvwqFWrVhUP5lihUEh47bX3TohExlvD4a3WYPBtVzi81ilJUy0rV26eWVtb68Ag4w61+tzrvTSUl3e+3+8/yvPSS0un9POSId/T7tixw3v55VddvGHDhoJ0H7+mpsa5Zs22Y5zOiyIu13+E166tnLpv3z5XwvExFBtJMpIE9ERaAHwCYDGAhZBnawcAnAHgJgA/B3AOgArFRta1cJckKaMFdIqbCMIYTCfWEARBGEzSLdxhziDZzL4D5vPbbP5mNe7j3R0F1xVUW0usqH2g9rzKcysvi+yOjM27LG9t/mX5Owd7PNc0l997unfPqD+PqrTkWuJMYChbWPaJ9xzvv10nunqtMh+ItqVtY0PrQ5Ns42wNzM3CDXc1nB6rizkGeNmgkwLhreHc0MehyYd/cXi+6BctXOJo+G3D7MCKwDc6P+ksGerxu4iDaV/NY0dU4WXzDPTBwiHPRV8N4CkAfwHwLwB2yDPTbwXwU8jiulFduXRv4S5JUka1cNfCGPMC+COA3wJ4CcAyAOr3tQVy28nzh+fsCILQGWrhPjBm9t1MfmddRTYBVFfXFOTktKG+/l8exhbwcLjBEQg86XI6Y9HKysrcwRzr0KFDzkik01ZQUBm2WP7AGfudxNjvJI/n4wBjcV5dXe0d+Cg9UavPc3LmBQXBDrf7ks4NG6on79q1y9PPy4Z0rT777HPTKysx/rnnXjgu3cdXq8/z8i7y5+VdEGhvL/X9/e9LtAsCjBSDR6qAriUE4EsA/wDwIIBXII/BmgngWgC/AXCm8lwjtCRD9KpMrkCnuIkgjINauBMEQegLtXDvH7POfzddYoQxZrb2i1kPExiKf1l80HuZ95j9Z+73ie2iL//y/I/HPj52UyrHsxZaYxUPVXzR2dk5Q71YmMBQcl/JF1qzSPL707a0baw687zorqKvorui3ob7G05XZ6LbymyRRJdSOW8A8JzuaeZ38NVNDzadefgXh+dbC62Bzi2dR+d+P3dj3o/y9g71+ACgzjxnNiYW3VL0YcuzLSc03tc4R52JrnkqCei90wpgo7K5AEyG3LZwCuQkzS8gtyvcCaAmzbZVjKhAF2DMbMK0wRgTOOcSgLMAnAvgGs753xljascAADgMeS7j5GE6TYIg9CXpFu4wbyGIWQV0CiCIjOcvf1n4yb33/v6s9esrJLf7T/7OzqhLkj60vPDCA8tGjx6dGJP0y6RJk0KbN699MRwOz4jFYiLnfId2v9Vq5ZAXiAJJfH/U6nOr9dJ4KLTVAQA2W2m8paUwr59Z6EP6LdqxY4d3w4ZdxwJz+FdfrZ+4Zs2ar+fPn6+dBZ/y8burz6+NWCweDgByFfozU/ft21eZMAtd7znuetvQw04cwC5lYwCOQvd4LAD4IeSOX+rc9FCa7GoxJN5UBHSKmwiC6BezBh4EQRBGMaCArmnnbcbfZLNWoJvVbzNhimQfFzlan23t+u3qWN0xPfhlMCfl46VxMY0lzxJxneKqVmeeqzPRraXWDsEt9LfSPKXPznuOt6HojqLVsb2xss4tnUfnXJGzueC6girNU1JObnCJo+G+hpPVmefu09wtpX8qXcfcLNr0QNOpUlhSPwOjxO1MFNC1dAL4N4DXAOwAUAdgH4AZAK4BcBuASyHPBLSl0a4RM9AzsQJd/W6Mh1wxsVz5uxVAnDHGICeBJACDqtIiCCJjUCvQqYV735jVd1P5Lf+TJ/9xOM+DSC+VlZWeN9/8gFmt10mC4OEOxy/DjY3hwpdffnniYOefA4AgCLBYLLBYLLBarVy7DfZYn3/+eZHVKkl2+1vxaPRuezR6tz0Wu8/m87UGd+3aV9rPS1O+T3/22eemB4Oj3cXF97dEIlMdS5YsnZHwlJTjgJdffnlSU5MnPxqtsdXX/7Wgvv6vBZHIPntLiyfvxRdfUgVFI79fmSaga+GQFxb/C3LcBMjjsdwALoEcM10L4FQAhWm0q3vMBGRsBTrFTQRhMFSBThAEoS9Jt3A3YRU2QEKy2fw2m79ZDY9zVvvr2qmB9QGh9BelK3IvyD1Y/b3qK/b8YM8VE16c8LrnOE9HOu3FamMOa6kVzNK16AhSpyRIAclqLbZGE5/vW+Cr8y3w1Wkfc0x1BEr/VLq5DxNDuj65xBFcFRyr/r3zs84K0S9WWXyWxH8HBp3cYAJDwc8KvhRyhZg681ydiR7dEc0RnIKRnUwytQK9P0IA3lFslkMWzqdAblsYB7AH3VUWwT6OkQy6t3AXRVHgnB/xfcgQ1O+KOqPSAkDinHPGWA4AH+TFDgRBZB/J/jaaSkzVoiy6NqXvMJffpliEazYeeujPJwaDeXC5vh+PxQCrdZoUjV6Al156/eSbb755p9Pp1EUs/OSTT/J8Pp9/+vTp/r6ec+2111bPnz//cG/78vPzY328bNDfyXg8zqxWK1erz53OWyOC4OQez/dDX311d29V6Cl9F/LzCzuPP75mF5AY8tlQWFioVksbUR1udAW63qh2vgDwPgAPujt6zYc8L70J3R29DiF133WPmYCMFdBVKG4iCIMgAZ0gCEJfkm1FiCSel42YVUA3Y2LEjD5nNYGNgfzg+mBJyb0l4qhvj9oBABNfnvh69feqr2h8rvFYz5OejemyJYUlof62+jNcJ7uKCv6zIMIsDFKnJNTdVncqj3Bb+dPla5iQtp+Rfq/V6L6oq/Xp1ulFvy36XBXHxaAo1FxRc5nUIXnzfpy3wT7O3qG2cx/1xKg1yvOGdIKu2a62xMdsZbaIrcyW2O6QWrinbodDTvYcArAKQAHkpNBUABcBuBjAQchJoUrISaJUbelChs5AV9+TSgBOyNX/j0OeV6/eI80GMAbyQgeCILIPauE+MGYV0M0aQ5jxs85KmpubbevWbfqGKOYhEDjDpnQLAudh1tzsL1iyZMm4n/3sZ3vSbJYFAgE88MBTszwe65gwNuGPAAAgAElEQVSXXnpmRV/V6YIgYOLEiam04U76u7lu3brChQv/cvo999zx0dKlr0wNBke7i4oubgEAt/uMcGPj1IIlS5bOmD9//mr1/FM4HwDA9ddfV3X99aga4GnZJqAbFTNBYysIWUz/AnLHrgmQY6bjAZym7FcXIO+BvCh5MLaM8oniJoIg+oUEdIIgCH1JRjShFu7mSxCY1W+z+ZvV+E73tYxfPn4jK2Rz1cfcM9z+ye9MfsVeYQ+n05bgFKScy3O2+d/1H9P6dKsj9we5lvq76k+J7Y8VF/6qcEOaxPOkDhKribnDX4cr6n9T7y5dWLqeWRmvvbb2vGh1dFzulbkfatq2r256sOnMtr+1TSr8VeFOGJdEyaYkzXDbaQHwibK50V1lMQ/A2QCa0V1lcbCf42ht6d3CXeCcDyZBNewoc/wAYA3kFpGPMMbU99vNGLsOwO2Q5y2+MjxnSRCEziTVwt3kVdhm9d2sfhNZgs/ni//sZz9clZeXd3lra2tla2vrAXWf1WqVzj777F6rv5Og3+/FqlWr0NDgyQECtjfffHP0FVdccTBFO4O2nciSJa/O2LvXO/qZZ56b+fXXNRNEcbSlpeXmfHW/JAUsW7cenrB58+bPZ8+e3YbsGEtlVOxnSLU2+vcnhm6xnAGoQPci5BOU/dWQY6bdGHhuupEz0DNKQKe4iSCMhwR0giAIfRlwBjrMK6Zq57+bznfAtG37iSzCVmqLxeM9tTrHWEc6xPMjAuacb+UctJZZm1ufay31v+O/DBZIhTcXrvee7W1Ig71+bWvxzPE0Q8TapoVNc+turpsHAZIUkNzFdxT/X/5P8ruqR7zneBusJdZljhmOxFb2eidqSEAfHMkmnUIAvlQ2K7qrLGZCnvsXQs8qi95aXhrRwj0TKykAAJzzGGPsXsif/f3oXmzwNIC3ANzCOR9KC32CIEYuycYEZhZTOUy44FqZCW6mz9ysFfdZi91u53fffffWQCBwmc1m2+9wOLbobbO5udn+zjvrYLdfFOvs3Gt5/fVl0y+99NJDqcxI74ekjrVmzZrirVubx9tsZ8e//nr5+Fmzxu9yu90xIKJ5lhc220xp1KhR2jjSCHFbT4xsrW5UzIQkbHEAB5RtJYAidI/HulSzX42bWvqwRTPQ+4HiJoIwDhLQCYIg9CWZmz5q4W4+383oNyWDshNDr2HXbFdn63OtsmEni3rO8DQO8JLBkLQvnjM8TTzCNzQ90jQXAApuKtiQc0lObeLznDOdWvE8W+aTZ5uAnkpVeBzALmVjAI5Cd5XF8cr+anQnhrTzFvVu4Z6xiSAA4Jw3A7hJSQidCLkd4VbOebpbmxIEMbJQW7gPBLVwNx+m9JsWWhMDwTnv83fz1VdfrTh40IqCgkuCgcBhce/eT8vTXIWe9PW5dOnr08LhY+xlZbc21dVVFwtCGA899NCmdB0/RaiFe2p2kIKtJgDrlc0LuVJ6KoCzAJwLoBHdHb1q0f2bTxXoA0BxE0EYg1kDD4IgCKMYsAJdU4Vtxt9kMwrJgHnFZLN9zsTg6fMakToloemRpiJmY9z1TVc17+SO+t/Wz+bxtCUYk04KSJ2S0PFWx2T174H3ApNEv2hJ1/GHAFVTGG+HA6iB3ELvSWVbA7nl+yUAbgNwLeQqdd2rKURRZJnWwr0P2iHPoX8PwL7hPRWCIAwgqRbukDUjs95PmtV3s8VNZvPXTKT7+9vr8err6+3vv79pgtN5DqzWXO5yHRMVxZMsr7++bHo8Hk/nOQx4rarV5z7fdwKMWeHxXBn87LNDR2/evDlvgJfqHQeQgJ6aHQzRVgDA5wCWAngIwGuQRfNZAH4K4BYAF0GuWtd70TEgx2YZK6BroLiJIHTEjGINQRCEkSRTTcEB064yN7PvgLkEZUoGZS+6f7ZSWBLqbqs7NVoVdRb8oqCt5L6SL/Kvzf80si1SUf9f9bO5ZNzlJXVKQt2tdafF9sdKin5T9FHxXcVrYodihfW/qT89CRHdCLIpSZOJdpoBfAzgbwAWAngbcgX6fAAOyEL62ZCr1tP+b4Dy72nGJoIYY1bG2CWQE2qfQU6yvcoYO294z4wgCJ2hFu4DY2bfzeq3WTDT56v7ve2SJUsm1dfbcxyOEvj9Gx2BwCdOm21ibO/eWPmbb745Ok1mkvrM1Opzj+fkMAD4fPNDodBE14svLp2WpvMYKiSgD84OkL7FwFEAOwC8CTlmegHAVshjsmZCXox8FYBvAHClyWYXmsUkFDcRBNEv1MKdIAhCX6iFe/+YvQLdbH4TREowO5PsY+2t3vO8Tc4ZTgbIM9EBIF4fdzMhLV+lpJIcgQ8CZbH9seLCmwvXec/pmr++tmlh09zA+4Hy3Ctza5I5Po9yxuysh63eHhsk1MJ9ZNkJAvhC2WwAbgfQBrnN+2nKfu3c9CFXjouiKEiSlJGJIMaYADmB9ksAdZDb4FsgLz64gjH2AOf87mE8RYIg9EOtQE/meWYtBDGrgG5Wv83oMzE4ev3NrK7eX+TzhcPAs4jHWwRRjFusVoiCwOOVlZUFANLVxr3f3+zPP/88d+fO+grOudDQ8NNC9XFJCrAvv2wYv3v37i8nTZrU14zmXu/Pd+7c6Z0wYULQbrcP9d6dKtBTswOdbEkA9ivbCgALIAvnOQAuU2zuR3er97ahGozFYuq9hO6z1vWA4iaCMA4S0AmCIPQlmZtLauFuvgSBWauxzfY5mwFDPlMmMBTdVrQNwDTIs70AdIvoRpLzHzm1zuOcy+1j7Z3qY54zPE228bYej/VCV9IhVhtz1N9ePy/nqpyt6ux0KSwJ9bfXn2Ibb2sp+nXRjhRPjwT0kWsnpvz/KwCbAVSge276Ccr+ashJod3onps+KCRJyrgKdMaYwDmXAFwN4BoAD3LO79LstwJ4AvKMv02c87eH6VQJgtAPtWsXVaD3jVl9N1vcZDZ/idTp9ffgsccWrgsEAgccDscPw+HwClEUG9V9JSUlET1taxk3blzoW986fUs0Gk3o0FUMp/OYeFlZWXiA4/f4LlRXV7t//vPfXjh37nGVv/vd775M5aQTjq83JKCnTieACIDnAPjQPTf9bADnAWiAHDNVQm4BP2hisVhGVqBT3EQQxkMCOkEQhL4MeHPJGKMKdHP6DpjLb0oGEYNhOK6XpJMCvQnlA4jnPbAWWaOWIkug9anWUwF87D3XW1d/e/0p0epomfcCb3Xyp3wERv2mZJOwLRhkB+j2iQM4oGwrIc/5mwpZUL9Us1+tsmhN1oAoikIGzkBXZ8N/E8DXAB4FuiorwDmPM8b+CLlq/3jIbfEJgsguko0JpCSek62YVUAHzOs3kV3ocR0fcQ/rdDolp9MZAwCfzxeFLETqQb/3zwUFBbGbb7451UXBR7B48QtT6+tzi9as2Sr86Ef7K8eOHZt07NULRorB2SKgqwU/RsWBap7UD7k9+WeQF9IfDTlm+iaAucp+NWbahyQF8QyuQKe4iSAMhgR0giAIfUnmZsy0IjJjzJS+m9VvmM9fYhBwziFJkptz7hnoqYackEKsJuZkdiZZS63RrhOIchbZFfE6pzv9gzhUV6KG2Rkve6jsk7rb605p+WvLaS3/0wJmYVLBzws2+C7w1Q3hdKkCPTU7RiVO+hLrmwCsVzYvuqsszgJwLoBG9Kyy6PN9kSSJZaCArvoTg/xedAKAUl2h0gGgBd2V/ARBZBfJtnDnMGfXLsC8ArpZ/Tajz0SScM5tnHMvZPFwOOj3Pn358uVl7e3t9u985zsH0nH86upq97p1lce43T8KtbW9l/P3v/99yhCr0KmFe2p2YJCtvmKmKIDtyiYAGIPuRcgnKvt3Q46ZdgPos8uBKIoMgMQ5z7RCD4qbCMJgSEAnCILQF7XSbKDnAOZMBplVSM60m3SC6I+krmdJkuySJOVIkpTLOc9V/69sOQByIc/tAoDjAGyFHBwfhDG/Fb0mBQ5dc+hSHuG2itcr/mEttUZ5lLOD3zu4IFIZGTd+7finLXmWwYqVHACYnfGSP5R8WnNFzeXggGuWa+8QxfMex9eRbBTQjaxAH0isDwD4XNnsACaiOyk0R9mvrbLocf0pFegZ1YpQc77/AjAdwDwA7zLGGACmJITOhuzrpuE5S4IgdIZauA+MWX03W9xkNn/NxoCfL+ecSZLklSQpMV7q2gC4NMf7BoBtkO8NVcHQCLGz12MHg0HLnXfee3kkEnHMnTv3f8rLy1OtgO86/uLFL0zt6CjzlpZe2dzWZvWtWfP8MWmoQu9hQwdIQB+arYHsSJBjoX0A3gdQiu7xWJeje666Gje1a18ci8WYphtoxkBxE0EYDwnoBEEQ+pLMDRm1cDef7xyQg+PhPhEDMWMyyAyfLwMAzrnQT6InJyHRo+JnjLUzxtoFQahljLUDiMXj8XLIQvp0ACdDXkG9HXJiyHAKbyn8sO7Wuu/UXFHz7aNeOeqfdb+uO6vzs85phb8sfGeQ4nmP60EKS0LDfzWcDAYwKxM7N3dO6Hi7o06diZ4i2VaBbkRSw0gBHYO0FQWwQ9kEyHPT1SqLWcp+7dz0zkybgc4YmwHZp04AbZCTW4sYY38DsEF5zjwAPwTwvwA+HaZTJQhCX9QKdGrh3jdmFtDN6LcZfTYFkiQ5tDFTYuwEeeaztrgiqomZDjHGtjPG2kVRLJIkyQ15FNB/ALgY8n3hNsgLLvWkz+vzkUcemd7a2pkH2PHggw/Oevzxxz8eyvHV6nOn88aIILh4bu4l/oaGtwqHWIWebTPQjYqZYKCtwdqpV7a1AHIgx0tTIHfzOl/Zp3b0Oqy0cM8oAZ3iJoIYHkhAJwiC0JdkWxGaTUxVMauArmI2v83mb1bAOQfn3KkmerRJHkmSjgLgCAaDd6Pn5xvRJHoOMsa2KX/uEAShXRCEjt5WfIuimCOKooNz/jWAdyC3ZZsGYAZkMT0OuVXhaACH0uxqr0kO3wJfHYBX635dd/XeOXt/BQCFvyp8u/DmwsHO9Os6vhSWBHXmeeGvCtd5z/LW191ed4o6E30IIrpRyaCRUnmQSXaGOjdQraLYD2AFgGJ0i+mXPfPMM/zdd9+NlJeXB30+31BaWhrNdQB+DnkxgB1yq0ERwF3o7kghKft/DnkxwVPGnyZBEDqTVEzAGOOcczN27QLMKySbNU42C1m1yJpzbpEkydfbomIAiMViF8RisYu1LwHQoYmbDgiCoP5ZjZnCcnFpT8Lh8ExJkmoBvAtZdD8Gctx0KbqFwSmQFyPrMQf9iM8uGAxaXn/9ndOAy8FYCd5//5mTamtrP0uhCr1LQF28+IWp7e2FOUVFJ7fE400CANhsZ8bXrHltKFXoRrZw15tsrEDvq4V7snQA2KxsDshz06cCODkUCs0755xzpOOOO655/PjxYIzZOefR/g42gqC4iSCGARLQCYIg9GXAmz7NPGwzJoPMKqBnVaIgSczoc0agJHpyequEAKBWj9s1L5HQnejp5Jz7bDbb8gSBPB1JGo5uwXA5gLGQqytyAPwU8orrbZCr09MtpvfAe7a3XsgVOsRGsQgAcq/OrR7C4bjULlmje6PF+Tfkf6y2bS97qOyTQz89dFZka6QQl2AoAno2VaBnQoImWdKddGpUtnUAfD6fb47D4Zi2fPnywlgsdhNjbC6At5TtsxE83+8pAKsgJ3osmo0pm4DuhJAPwJphOEeCIPQnmbFX6vPMFjeomNX3kfrvl16Yzd+MQVlU7O5tUbFGJPcmvKxTFccBQBCEXRaLZbsqjguC4Nfkg4aCH3K75k2Q75dOA3ASgLMAnIHuyvRKpEdM7/W3SK0+t9luiTGWi87Oxa4hVKFzANiyZdsEm80Va2v7iS8YrHU7nfkRq9UtRqOwf/DBB6Ovu+66qiGcP7VwT56hLgYeDOn0KQL52t8GQGhubp5y4oknztuwYUNJbW0tA9DIGFsGOWZazjlv7+9gwwzFTQQxDJCAThAEoS/Uwr1/zFp9b9aFA4TBcM4Z59wtimKvLQKV2eOJiZ6QmuhhjO1RKyAYY6o4HlATPeFw+NR4PD7H4XB8lu5T7+Xv+yC3XmsA8AnkCovjAJwKuYWZ2uZ9qNXbPWyrM8/FJrHQc7ZnU2h9aGbNFTXfVmeip3B8SAHJChGW4KrgeN95vjpmZzy4LlgkNos5PMxbUjz/rlMe4usHIhurKYwU0PVoFei/+uqrl1199dXLrrrqqgvWr19fDbka4SYAdwP4NoDXdLA7ZDjnaot6giDMDbVwHwCl+t6Mvpt14YAZfR5WOOc2URRz+mmtnoOeeXQRgBozNQqCUKWtHrdYLB2MsZhybASDwXstFkuVw+FI51iq3u5h/ZDjopMAvABgFIBjAVymnHMV5JhpF4Ympvewra0+F4TJHAAYuybVKvSu6/+BB+5eWV9f7/r000/L3norctK4cdban/70u5sFQcDcuXMbh3D+R/iQZrJNQDd6BroeMZNUUVGx4/HHH9/xxRdf5Fx66aU/i8VijwC4BMArALYAmK2D3bRAcRNBDA8koBMEQehL0i3cYcIgWbPa2nS+K5jNbzP5a0j1iCRJdrV6vJ9Ej0XzkrhSAdHOGKu3WCy7tAK5xWJpZ4ylPNc7DQx0PHX/PmVTK9MTxXR1lfnhoZ7Qwe8fvECdeV54c+EO/zL/DnUm+pj3xrxiyUl6DnpX0sE+0R4quLHg45ZFLafW3V53iu8CX3XzE82n2cbaGovuKPr3EE6XKtBTs2PE/DtDqjYkSZIAfMU5/xNjzArgdABfpNsOY+ynAJ4BEFAW4qT7+Dblj6qoxkdwFT1BEEMn2ZiAw5xduwDzCsmAifxOUzUykQDnnEmS5O2rclyJm1wJLwtoWqvXadqqq7FTqLfW6gbS37WinlgAwEZly4EspE8DcDl6iumVkKtak+UIx+Xq81AeYxN5PP5/FgBgbLQUCknuFKrQu+KAE044oT0ajXY8//yrs+LxKda6ur1FDodDOvvssxsGcbwBz18HSEBPHd07hImiKEiSFOOc3wfgPsZYBeTFJmlF75hJsUFxE0HoCAnoBEEQ+pJMUp5auJsoKaJgxptZuocfJJxzIYlEjzPhZX5Noqc2IdHTMQISPelGArBX2ZYBGAc5MXQC5NaFrehu8z6QmN5rUsB5grPGfap7rzrzXJ2J7n/XP0nwCoNZbNAD38W+wwA+bv5r8+mRnZHR9rH2hrI/l30seAQx1WOCBPSRbgd625IkiUFOiIJzHgfwYbptMMbKAfwZ8uiE3DQf2wu5CmQe5ESv6k+UMVYI4E+c81TagBIEMbJRk74DYWYR2ay+UwBB9IvSWt05wKJiH3rmW6KamOkQY2x7QtzkZ4wN5Z48kZHw3e1At5ieiyPF9N3orkwfSEw/4v65qqqq2GIRReCP3U9i4FYr4nv27CsZyom/8cYbFfv3i6NKSm5tbm1dlPvKK29MO/PMMxsEIeUUGrVwT80ODLSlq514PM4YY135Ws55DYCadNrQM2ZSjk9xE0EYAAnoBEEQ+jKYGegjIagyGrP6bla/CQVNoqe/GXo+9LxGIppETw1jbGsviR4jKmmHm75+VyUAe5RNFdOnAZgFuQpXFdO3AahL1ljxncVfJz7mW+CrU4T0wXBE0oG5NdX+NojMNuTPz6hqiqxInAyDHUDnandRFLsEdB15GrIw3wbgW+k6qFIx/xKA8yG/XwHI71eB8mcngGfTZY8giBGFOgN9wBbuJm1jDphbQDej32b0uVc45xZFHM/pZ1GxXfMSCT0XFR/QtlZXtnTMAc9k2iGPxPoEQB5kMf1YyPd1cXSL6buRZGX6888/v6q9vf2j3vb5fL7BLjruuj+PRqPsjTfenyFJpwpO5+RYTs63A9u33zNu9erVJUOoQicBPTU7gDGdu4wQ0AVkaMwEUNxEEEZCAjpBEIS+JHvTZ9bEgNmFZLP5bSp/OefuWCw2LrESAoD6f5vm6RKADiXR0yoIwr6ERE/HCE70GFkZlGwwrRXT3wMwHrKYfiJkMb0F3WJ6vebYSPL4qdDj+IFVgeLmR5vn2CfY67xnefe2/q31pLrb604pe6jsE2YfUvtOIz4Poz7zbBTQDatA1wPG2Pchf4emAfjvNB1T4JxLAH4O4BgAEwA8AaCGc34zY+w4AC8CWMI5fycdNgmCGHFQC/eBMWu8CJjLb1NV3CsdylgsFivnnPcqkAPwJrysUyOO79X8Wd0CI7gVfjrPq7/vxWDuO9sAfKxsqpg+DcAVkMX0XZC7ee0CENMcv8exrVYrLywsjCE9dB1frT7Py/tOBwC43bPDHR3TvWmoQgeyI27KqlhGY0v3Rcd6Fh/oETMpx6W4iSAMhgR0giAIfUn2hoxzzs2YDDKrgG5Gv0dqEmPQcM4Z59wtimKvLQKVRI9VFMV5oijOU14WUpM7jLFqzdxxNdETHMGJnv7Q6xru770Y7PskAahWtnchB5nHApgNYA6AZshC+lDm6A2KaHXU3fxo8xzbGFuj2rad2ZnYsqjl1MYHGo8r+X1JqjOrs6WFe1bN2NPYgd629BTQGWOlAB4FcCfnvDaN4yDUA30TwHrl2LkAahhjjHP+JWPsTgB3MMY2cs7XpcswQRAjBmrhPjBm9d2sfmeFz5xzmyiKuZIk5fTRWj0PgBCJRK5TXhJH96LiRovFUqUVyC0WSwdjLF0iLXEkWjE9H72L6dtg0PUpSRLeeOP9GaFQsYuxNdzvX6M87mTbt7eNW7NmTfFZZ53VmMKhqQJ98BgSy2hs6d7CHZkXMwEUNxGE4ZCAThAEoS/J3vRJyJIgeZCYUUhGhgql6SAjPmdJkuwDzNDLAWDRvCTOGGsH0M4Yq7dYLLvi8fgcQRA+s9vtnyqJnpRnZRM9GOo1JAGoUjZtZfpJABzKc2YB2IT0C+pdSRT7RHso93u5W3IuyTmkzjz3Xew7DBvWOyY5/EO0kQ1JmqyasaexA2R2C/e/AtjOOX86zcdV35siyIlZQP5M7JB/a+MANgMoATAqzbYJghgZJN3CPYnnZCtmFZLNGjeNeDjnTJIknzKOKqeP1uquhJf5GWMdiiBeJ0lSCed8lNPpfElZXBxKs9g0UtDDKfV3sz97Q/n+tALYoGz5kGOmYwFcqRy3A3IFbBW6K9PTRdf9eVFRTmDaNH81sLzHEywWt2SxWFL1z6ixV0B2xGaqHaMwpIW7jhXoesVMAMVNBGE4JKATBEHoy2BauJuuAl0THGdllJwEZvV72OCcC5IkefuaO64kepzalwAIaCofahNaBLYzxjoTEz2BQOB0QRCarVZri5H+ZQFGfidEdIvp70Ju734+gBmQRfQmdLd5T6WyoS84AORdnXcgcYfvfF/9kU8fFNkyAz2rWgRq7AAZWoHOGLsCwIUAZqb72Oh+TxrQ/ftbDWAcgNEA9gMYC8ADIKyDfYIghp9kBXRTxkwK3MTz383k94hYMMA5B+fcqRXH+1hUrP1sopqY6RBjbHtC3ORnjPW4R+ns7JwvimKR1Wo9bKiD2YGR34tWAOuVrQDADyDfs10FWTyvhNzmfTdkAS8dcEEQsGjRE73OVR8iVIE+cu2otjJy0bHOMRNAcRNBGA4J6ARBEPqSdAt3mCsxoMWMvnNAXrU/3CdiILoHW0qix6VWj/fRWt2HntdbWFMFUcMY29pLoscIcS2TMTrRp4c9EcBB5c8vQL5OjgVwMoB5kAX0bZATQ6mK6dkkbuttw8gWgVnV9lAPAZ0x5oE8W+8JAPVKm0AGudIByt9jnPNQiibU92QTgJMYYyUAXgPwKoDHGWMrANwAwA95wQtBENlHUvdaShcnM90/azGr72b1W1efOecWTczUVT0uSVIuADVusmteIkFura7GTQcSxlG1C4IQSfF0zPT5jogFEkOkBXL1+QEAH0GuTJ8GWUyPorvNexVSF9P1viaMFND1xkgB3ai8iBEt3AVkXswEUNxEEIZDAjpBEIS+UAv3gTFjUiQbAudUGNLnnJDo6dFeHd2JHpvmJWqip50x1ioIwr6ERE/HEBI9hIzR310j7EmQEz+7ILdBmwg5KXQqgDPQLaZvg1ylnixGVSFkg4BOLdxTRBHQ022jCEApgFsB/KaX/a0A3gRweSoH55yr5/tPAHsACJzz1YyxhwDcAeAS5fEbOOfbU7FBEMSIR61AH6i6nGIm82G2uGnI/iqLij2iKKrx0hHt1QF4E14WUmKmDsbYXu3ccWULmHgMWSZi1L10C4B1ylaI7jbv34YspldCjpmqMTgx3aj7c6pAH3l2DLEVj8eZDoUSusZMAMVNBDEckIBOEAShL4OpQKd2hObDTH73GwApM/Q8fbUIVBI9noSXBdUqCMZYtZrk0SR7gpToyUr0+kx7S3KI6BbTrThSTG9At5jenKQdvRM1RonbRtjIpmRQJrdwr4N8vSdyF4C5kEcfJHv99wnn/DDkcQrq3xcyxh4B4OCcdw71+ARBjGgkUAv3gTCrgA6Y1+9e4ZzbRFFU46WcXlqr50JeBKoSB9CuxEyNFoulSiuQWyyWDsZYumdYEz3Jtmu4t/vnZgBrla0IspA+DfJ4rCiAnZC7eVUhuXvVTK8OJwF9aLb0buGe9gp0GBQzARQ3EYSRkIBOEAShL8kmOsycEDGj72rgYRq/OecCACEajU7sZ4aeNtETY4x1QE721Fssll1agVxJ9KRrvhphEOE9YZfFbRFtZbao+pgUloRwZdjjnun2a57aV3A+nN+ZOOQqikrI99BHQ04MnQZgPoB6dLd57y0wphbug7MBA+wABrQI1NiB3rYkSUp7MohzHoGcDO0BY+waACLnfF067R1pnncyxsYAeBrAvZzzTTraIwhieEj23tiMcYOKWX03ld/qwnJJkooikch0znlv1eOuhJf51epxQRDqEirH2xljIcZM8xZmHW+//XZZOBy2XHXVVYd62T1QzDTk+854PM5isetvNuMAACAASURBVBhzuVyDETSb0FNMV9u8fwNABD0r03u7b82mFu7ZEJsBxsVMhtiKx+NpF+mHOWZSToHiJoJINySgEwRB6Au1cB8YUyVFFLKqKppzLkiS5EucoZeQ6HECQDQa/T5k//2aGXq1vSR6OinRkzEkdT1ziWPfDfvOhQjLxJcnvmsrs0WlsCRUf7/67Oj+aMnU1VNfgVvfE215tuVoHuVC4U2Fu9THgh8GiwIfBMaV3FfSwGwMSM6fOOQqip3oFtOnAZgD4EzIq8+3Q04MtSivMSpRkw1JGqpATxFRFPWoQO8PXf3hnKvH9wA4HXJVHUEQWYbP5+N+vz+Z3xPTxkwmnv+eNXGT0lrdqRlH1Vv1eA4AxOPxM5WXRRljbUqsdIgxtj0hbvIzxoz8d58YGoO6ntvb26333POn82OxuG3u3Ll/Lysriw78qvRyzz2/n3X4cFPuc889udpqtWoXOyXrSxPkWekfASjGkWL6Tsgx0x5038PqfX9OAvrItaPaysQK9P7Q/b2juIkg9IEEdIIgCH2hFu4DY9ZkEJABfiuJHpcqjvfRWt2Hnr6ENVUQNYyxraIojuacH+VyuZ5TEj26BkSEYSR9DTOBYdQdozbU3FJzcfX3qi+a8MKEZft/vX9u57bOcaPuGLXC4rOIivg3ECkHnzzIbYF/BY4BgMKbCncFPwwWNT/WfJqtwtbM45wNQkDXohXTbehdTNcK6TQDPTkbMMCOasuI3yNDZqArlWuGJIM459cAuMYIW5DfP4bBzc8kCCKzEEEt3PvDlDGTsqA2I/zmnFvUeEkVyNXOWwDUuMmueYkEQB1H1S4IwgHGWDAej59ntVpX2O32LwRBiAyPN8RI4PHHHz+mqQkFnDvZI488MuOhhx76TLNb9+/FV199lfPxx3umRKN229tvv11++eWXa6vgU7lPbwTwobJpxfSZAMLobvOe6vEHCwnoI8+OaktvAZ0ZtfjI4JgJoLiJINIKCegEQRD6kuwNppnngJsxGTRiWrhzzq2aRM//Z+/Nw+S4yjTf34msRaWSVFpsy7ZszGKz2Ja3xixuDAaMDTZglmZp6GaYGe4w2+1mGmbm6Z57p4eZ3gammelmZp5p+vaAGZpLw2VYbQMGDM1itgYs2/ImWdYuWbJqU+0Vce4fJ0IZSmVlRlVlRGbEeX/PE49sKTNOnorIqPN97/ne77T+49QTPf2pt0TUe+iNBkHwREPf8YlmiZ6ZmZmXh2F4fq1WGy9qbj1A169vrzHyypHjfISv7Pvtfbc9fOPD7wY473fPu+vsd5+9N+MpVvUz3fK+LQ8BnLzr5OXzj86ftbB34ez+C/uf2vonW+8NhoJtqzl3zALwUHz0A5fgbN5fSv17dA3wU2C0A+M1UlRio2oCuizce58kEVTFuQkhHFl7oPu6vvJ17j0x73hT8XAYhiOpeGlDw6bidQ1vm07EcWPMnnTf8fg4GTsLnCKKosHFxcWbjTGTEs8rxbLv4fHx8b7Pf/5rv2LMbdaYddFXv3r7Nb/zO0fuz1iF3pG19Kc+9ennTk09Y60xW+3nP3/XZa9//esPxVXonfhOpsX0c6iL6VfhPvcQbmPy43ReTC3ymVK4gB5FEVEUmZRjQKfGKaoIIncL9y64dhWJ4iYhOogEdCGEyJesC0xv7QjpkaRIwRSyc9daa6IoGk4E8Wb26jh7pzRTSfW4MWZ3uu94nOiZakz0CLEc1v/q+hPBcDATjoX9ABtfvfFwkeNved+Wh+YenTtrfvf8uQBb/2TrvcFwkA4uO3V/L+AqKHbixPPrgBuAa4EXA4eo27yPdWjMylm4j/4/o8+yoTWb37t5V/KPU9+f2nLyrpPPOOc/nPMzE6z610elLNyjKKpqssTgkmmqpBCiumR5dilm8o+i4qb+MAxHloqbcFa4tdRbFqlvKn6yVqvtSgvktVptwhizsJKP0pEJlQff5puZpPp8ePg9U8YMMTb22fVNqtBzY8eOHRt+/ON9l6xb9/7pgYELFx5//OfnN1Shd/LaPRkf9wBbgXfiNqS8E1eZ/hAuZtpDZ0TcSlu4/+7v/t4LDh06NnL77X95dxB0zLSl6Ar0vAX0qm46BsVNQnQUCehCCJEvmSvQkR2hT3SkAj2KooEMPfTSiZ4FY8w4LtlztFarPZoWyONET16LbCVHqkvma5v0PA8nwnUbX7/xpxPfmLhi9zt3vzbpiZ7hFKsOpqe+M3XW4qHFzcn/j35i9FlxT/Q8n0MLwBPxf/8VsAVXYXEDcCNOTH8wPlbj0lDEs7RQAX1+9/zm6R9OPwdg83s375r6/tSWYx88dmPt7Np4dDLqq22orfaZVbSAnmvlRtkq0I0xG3G/p0LczyY5bPwn1to53CaUqm4OEEI42sYE8SZOxUx+sep5x5uK16fipmbV40MNb5tMCeKH49ZUp6rHjTHTsb18Xvh4rX0g05ozXX3e1/e0CCAI3rqcKvRVi7dJ9fn557/imDF9jI4+f32qCj3P+/MoMIlrg/UT6pXpVwMzODF9J6sT0ysroO/YsWPDPfc8uH1hodb3pS99adsb3/jGgy3eu+Jxcib3sRYXF0sXVyhuEqI7SEAXQoh8yRrw+5oQAQ/nnkq2LDlva23QkOhpVj2+Jv0WTk/0HGqwCBw3xszknOhZkjL1LxTLIvM1tZHl8d98/JVJz/Oz33323vFXj+9NeqJf8sVLvsTa+svz+LDpnudb/2TrvaN/Ofrsk3edvBxgyz/bkthk5hWsJz+reepi+QDwbFxS6OXAq4CD8b/tZGVieqUq0M/5w3N++uTvPcn4p8dfNP/Y/Fmz981eVDu7Nn7+fz3/Wx0Qz6EAi8CYQpJo1loozl6xE3wW+BVggvq9lSSCLBAZYyJcsmgAlzgVQlSTLNXl3sUNKXyee6uYCWvtmiRmWsJafUPDOeZSMdMBY8zOhrhpsqjeuEI048/+7M+ed+xYtKWv75zFmZn/vQYgCDbZ0dHaSEMVei7rynT1uTFOOti06a2Tjz/+s6QKPbexY5I14dH4+DZwLnUx/RrqYvqDuI3KK1n/VkFAPy2W+au/+p+XT09fOATn8OlP/+8rb7vttoMdqkIvKmaCYnqgB3mPkQOKm4ToAhLQhRAiX5LFTEviRY6vCRHv5m4ThSOKNs/Pzw8uYa2+ntN/LrOpRM9+Y8wDTRI9vRwA+FaB7tt822ICw7qXrNs7cuvIo0nP86Qn+sS3Ji6sra+FcS+ydqz4Zzt55+QzT/U8Hw7CpCf6zA9nnhG+K9xVW19rd4rV0CyJMg88EB9pMf2VwE3AAeo27xMZxyhC3M77WXPqZ2UCwzl/dM5PD7zzwJaZn85cDHD+fz3/W7XNtZXYoi41lnqgd49fAtPAHC42DZY4Ilw10mocGoQQvU2W3y1JhZXxsKWQdzETnNoY1r+wsPD0dPV4FEUjQBI3DaTeEgETcdw0FgTB3iAIJhripp7tLe7hfe0Ly/ru7tu3f2TNmrlZ+PNTf9fXR9TXx+KBA4c2dPzTNfDlL3/5GbOzg4NR9PFgauoTp9wZrO0z3/jGPc9605velPezqNn6/Eh8fAs4DxczXYoT06epi+l7af/7pMgK9Lw59bPasWPHhnvv3f28oaF/NdvXd8Hirl3/4sIOVqEXWYGuHujNUdwkRBeQgC6EEPmS7ARst9iUHWGFsNb2RVG0oZm9OvVEDwsLC6+L3xJR76F3IgiCJxr6jo8HQZDF3lqInubc3z730ca/G3nlyPGRV44cz3iKVT0rzvngOT8jwqR7nm9535aHwhPhrtr62rb4r7qVuEyL6YM0F9OTyvSlxPTK9UAHmP7h9JbweDiS/OP458YvSvdE78BYRWw+KsrCvVTJIGvtv+r2ZxBC9AxZK9Ch2ER+r1DFmAlr7XAYhknf8TPs1XF9kJmbm3t6/LbpZFOxMeZxY0yjOH6yIiJ0pa61WB4f+9hffH9xcfEHzf6tr68vy/29KoH4da973Z6tW38y1ezfLrnkkjHguSs9d4c4HB/fpC6mX4arzp3GxUs7cZXpzT5nUW2vWGL8To9joV59fvbZt56APiYnr1nXwSr0Slm4l7EHuuImIbqDBHQhhMiXJFGeRUD3NUgu1eaBuIfecJLgWcImcLjhbVOpRM8uYD4Mw5f29fXd2d/f/1AQBFMVSfS0w9d7vOp09N6NK41yIRgKmoqXcTVzEZUUkO3nNQfcHx+DwHNwSaEbgZuB/dTF9MkmY+RJoQmndM/z8z563ree+vBTV41/evxF4Hqid2isKlm4lyoZZBr6itg8HwBCiF4nZHkCum+ULl601vaHYTjSEDedip2AEZzVbMIi9U3FT9ZqtcfCMHyOtdauWbPmf9dqtQljTKccaHoV/R6sNpmubxAEDAwMtHxtnkumK6+8cuLKK69s5X7VjQr0pUiL6edTF9OfD0xxemV64++QKli4G8Cmq8+NcaYc69f/5skOVqEXtem4kLHKWIGuuEmI7iABXQgh8iWpQG+Hl5Z8MdZa2zNzj6JoIFU5PtJor47roZdO9CwYY8ZxyZ4jtVrt0YYqiAljzGl9esMwHJmZmXlpEARP1Wq1k0XOr4tocV9NOvrdzRADFiF29tr554Ad8ZEW028CXg3so15lUbkK9LHbx6441fN8c23hnD9yPdEnvzx55cjbRp6obVx1H/SqWbiXKhmkxI8QIkXWmAncM7U0z7oO0VMCurU2iKJoXTpualI9PtTwtslUS6rD8Z8TSdxkjJlu0AeYnp4+Bxjs6+t7qqi5CVESWj0PinhW9FrMBHAoPu4GtlG3eU/E9KQ1VhHr8kIF9M985jPPnpkZHIK/trOzfzOYDB2GA7UvfemO53ZIQK+MhXscM602jiwUxU1CdAcJ6EIIkS9pC/d2rytNFXaHKSwZFCd61jckehoF8jUNny2d6DlojJlIiePjxpiZxkRPlo8S/9kzSTAheoxuBIdFVaCvhrSYvoYzxfS5eJx1QF6bcwoV0M/98LnfI8IkPc+TnugL+xce6IB4noxVZAV6rtUUZatAF0KIFJkt3OMe6Pl/ot6iyJgJa+2aJGZqJpDjNhWnP89cKmY6YIx5sEEgnzDGrOR3YE9tHCgQH+dcWbpYMJDXGrcMcdPB+PgGdTH9MuBaYDZ+zTZgF/n8nAoV0G+66aYn1q5d28Sh4zwuvfTSrO3S2o7TgfP0xFhltHAXQnQHCehCCJEvaQv3VvRUFXbBdCQpEid6hpoketJVEOsbxppNJXr2xf+dFsgnV5joyYpv19y3+VaeLm2CzjsRVJbzzwL3xccaXC/Cm4ELgPfjbAofxNkWdlJML1RAbyaSm8AwcNHATCfH6cC5emKsKIqUDBJClBVZuLcgbvfUkXlba2tRFG1ocN0asdZuAJK4aSD1lgiYiGOlsSAI9sYbidNx01wnPtsS+HS9VWEospLV7TDP8fOi0/FGIqYnlenXA88G3oGLk5LK9P0dHLdIAT264YYbjt9www2dEMqXIveq8BSycBdC9AwS0IUQIl+yVqB7beFOhrlba/uaJHpO9c+L/+xPvSWknug5EQTBnrjyIW2vPp/TnNrhY/JPyaDq0slr2+47UYXvTB7fhVnglzh7wqeAPbgKi1cDtwBPUBfTpzowXmECekXGgYIs3FWBLoQoMW1jgtTGVh+du7LGTFhrh8MwbOw7nq4eX9fwtulkU7Ex5vEkXkoJ5CdjAb8bqAK9wnjqJNEpsli4l7UCHfL57BY4gNuA/GzgU8AlOJv3FwCT1MX0Ax36DFVor1XkOFCAWK8KdCFEViSgCyFEvmTdFey1hbu11oRhuK6hAqKxeny44X1TqUTPrsQiMCWQT3Ux0dOOXv1ceeNdhsQDqnRNq5BoSnq5/TI+hnCV6ZfhhPRbOL0yfSVieqEV6AWNk2uFQ8NYeQvopaqmMMassdbOtn+lEMIDsjyLfdyEmmABE0XRQLKpuJlADowAtdT7Fo0x48C4MebJWq32WNp1q1arjRtjerkPrK9xkxC9ThH9w/M+/wFgN/B1nItXYvP+QpyY/iBOUF+JmF6ohXvOYxQ5TiFjhWForLW9/LvvDBQ3CdEdJKALIUS+ZLZwz/Ca0hInehotApMkz6Yoil48MzNzXeotC6lEz5FarfZIkx56pVrsLkFlr7kQOVFk4N5pikqipM8/A/wiPtZyppj+BHUxfTrj+YsU0PMWtrth4Z7rnKIoyt3ysMP8B2PM3wFfsNYuaf9rjDkH+BDw29ba8cI+nRCiSLy3cLfWBlEUrWvSd3wkiqKtANPT07/b8LbJVEuqQ6nNxBNxFfl0Bap8Sz+BZVDWde5q8OH6Fj3HMvQob3f+Ir4LNvXn/vj4OnAhLma6HHgRMMHplelZKNTCPecxihynkLHitldliplAcZMQXUECuhBC5EvlLdzjRM/6JomedPX4mvRbSCV6rLWzxpij/f39P06s1Y0xMxVI9CxJqjK+upM8Ex+TQaJcdEPg7jStkk3TwM/jIy2m3xofe3BJoYdpLaZXsQJdFu7d4z3A+4EPGWP+yFo7scTrRoDfBP5lYZ9MCFE0WWOm5FlXKmJr9TVJzNQokMcbizdw+s9gLuW4NWqtHenv7/9CEARjsUA+kbK1ryqV3mjeAh/nLDpPnuvOssdMrcbdFx9fA57G6WL6OHUx/WCGMVSBvnyKsHAvlWtXjOImIbqABHQhhMiXiBJbuMeJnqElEj1JD731nB6AzKaqIPal/jupHp9MJ3qmpqb+SRAETw4MDDxS+AS7h69ishJB1SSv/nRFjlcERVm4Z/n5NIrpz8Mlhl4bH4mY/hCuin0lY6yGKgroRVm4l01At7h2Ax8ArjHG/H1rbbNkZLJGqrpQJITPZHl29ewmVGttLbFWb3TdApK4aSD1lgiYSMTxIAieiDcSpyvIT1WYzc3NXbmwsPD0gYGBBzwQzdOUdd23Unybr1fk0GKuW/dLFSrcof3Pz+LaX+0F7qIupl8BvBgnpic2743r16J+T1VRQC/Cwj0om4U7ipuE6AoS0IUQIl+yVqB3ZWe9tbYvDMMN6QRP/OcGIEn49KfeElJP9JwIgmBPKtGTCOTzy/0Y9GASrCB8mreSQdWkSnaERfVA77V+gdPA38XHMGeK6Y9Tr0yfoVoCeiFV4TGFWLiXUEAfBt4HXA38e+BeY8wbrLU/b3hdgPsZlmluQojlkcWRqysCurXWWGvXhmHYrO94Ejuta3jbdKp6/PFEGE/FTSeXKab17OaBAvBxzkKshiLW0t1y1erU+VnmGI1i+kW4mOkq4DpgjHpl+qEVjrESihK2c68KT1GEhXsZ4wrFTUJ0AQnoQgiRL1kTHR23cLfWmiiKhpv0HU9Xjw83vG0qlejZlUr0JOL4VE67pn1LivicABOiFfpOrJ7VPKOngJ/FR1pMfz11MX0YqK3yM7ajihXosnBvjgWGrbV/ZozZAXwC+LEx5p3A5621yVwK2YAghOgqbWOCVOV1R527oigaSKrHmwnkOGv19O++RWPMODBujDlaq9UeTQvktVptwhjT6co2X+MHH2NF8HPOVSaP69nNe6Qq9+dK1+UWeCI+lhLTj65yjKxUsQI9d7E+iiKTijPKguImIbqABHQhhMiXTBbusSi9rERQnOg5wyIwVQUx0nDOhVSi50itVnukwSIwj0RPFnxMivhaje3bdRb5UNYK8aIq0Dt1/rSYvo66mL4R9/tlmHpl+myHxkwoWkAvIrkgC/fmGGAewFp7jzHmJcDngM8Af2yM+ffW2jnq65kyzU0IsTxysXC31gZRFK1bou94EjcNNbxtMtWG6lAqZkpE8hljijcPi//0bT3tW9zk23xFeSlLTLPU+enQGBGu/dUe4E7qYvr2+N//MfAArjr9cAfGa6SKAnohFu5A2SzcFTcJ0QUkoAshRL6syMI9TvSsXyLRk1SPr2l4fzrRc6BBHO9WoicLPgvoPs3bx2SQL9e3yGtbhZ9pTyebFo8uDvRt7Ztv+Lv5vq19PwV+CvwTnGA+ANwGvA7YjRPTH6EzYnoVK9Bl4d6c0ywarbX7jTHXA7cDvwtcbYz5Deo/vzLNTQixPDJbuFtrTfwn1to1SczUTCAH1jecdy4VM+1P/XdyTPZoj/FTc+/RmC4vvIoVU9fWmzl7RpECZJ7jFXF/9lrbq6bMzMwEX/ziFy94y1vesr+vry8tpu8D3hj/968ALwFGcTHTg8CRDn2EIgX0on435j6nklagK24SogtIQBdCiHxJFjdpcRxr7VA60RNF0VlA/9TU1D+MqyAaEz2zqeTOviaJnpM9mujJgk2SYL7h4bx9m68PdOOalrVCvIif1arGGP346DNH/2L0hq3/ceuXh182fBzg5N0nzzn6fx193Zb3bbl7469v3If7+RzFVVisp16Z/gbc77xd1MX0uVXOo4oCuirQz+S0z2utXQDeYYx5EPi3wE+Afxf/c1nXOkKI9oQ0/B6z1tYSa/UoikbCMLwQYG5u7vVzc3Nr47hpIPWWCJiIY6XRIAieSPUdTzYWr/R3U1dJtdHycT3t45x9wcdN1lWgqArxvM+/6jl86EMfuuJzn7vr+qmpqTvf85737G7ykjuBrwLPwMVMz8eJ6Seoi+lHm7wvK1WtQM91zR+GYWCtLVsFOihuEqJwJKALIUS+2IWFhfMXFxeviqJoGEiqx/tTrwlxC5tFY8xTQRA83iTRM9/07NXAq6qCGCUKhGjNUt+RMj8ret7Cff0t6w+M/8345JF/deS2cz907pfsvA2O/pujr+87u+/EupvWJZaD6TEmcUH6T3Bi+qW4xNAbcb/b0pXpyxEsqiigF9UD3VAuAb2PJX4m1to/NMbch6uquD3+6zImuoQQGYiiqH9+fv6qKIquSFWPDze8LHE5CYMg2J3uOx7HTVMpoblqeCmgG2OstVW9pMIj8vretouZyvrlKYWF+4kTJ/q/8pV7rp2c3Lj+05/+/Ave/e53P97X19f4rLa4fN/u+EiL6dcC1wNP4WKmnSxfTK+igF5UD/SyxRWKm4ToAhLQhRAiX2zcc+9sY8yoMWZXOtFTq9XGjTHTMzMzv2atXbN27dovdvsDdwHvBHRPK0jKGryvFN/m2ym6+Z0oKtHUs+fv29o3f8HtF3zpwN87cNvh3zr8VoD+C/qPbPvktq/0belbaDPGJPDj+NjAmWJ6Upn+KO3F9KKuRSGidkyRFehlqjY4AEwv9Y/W2q8aY64Dvgg8x1pbprkJIZaBMWYhDMPzgWPGmKO1Wu3RhrhpYnFxcevc3Nx7BgcHv97X17eair0y4mP8AB7GijE+zlmUhyLE1J6Kp0+cONG/efPmdDzERz/60cvGxmoj69b9weSBA//ivE984hPPbFKF3jiPtJh+B3Ux/QXAS6mL6Q8CT2b4aEUJ27mL2imKsHAPSmjhrrhJiC4gAV0IIfIlGhwcfHhwcPAHOHFhydfhb5Dsa1IE/Ju3b/P1haKTGz2VTFkGRVm4t/z5THxp4vyxT4w9f9snt91RW18LAeYenRs+8i+OvHrrf9z6jTWXr5nc9A83/ejYHxx7HcDmf7z53gbxPEtCYwL4UXykxfQ34cT0x6iL6c0cVoquQC8iuVBIcqtsFu7W2qdleM0jxphrgasK+EhCiC5hjJkfHh7+Gq4Cb6nXJM/rYKnXVBhfBXTwb85lXeuK9ujaZqMoC/dMfPjDH77sE5/4X6/81Kduv/3qq68eh3r1uTFviwYHb5mfm/vCYEMVepZ4JtlkvAtXmf5MXMz0QpyYfpy6mH6sxVyqVoFehIV76XqgK24Sojv4GHQIIUSRWLIJxBZ/n8k+Cug+Bs4+ztkHiv7u5jleUT3Qu9ovMJqK+ud2zl184K0H3hhOhrW5R+eGD/zGgbcvHFo4287a2sm7T55z/MPHb+47q2+0tqU29uQfPHnL1HenzmoYYzlzSMT0vwL+C/AtnN37m4F/CbwNuJzTe9hW1cK9CAG9RokE9KxYa09aa7/f7c8hhMiVLBuKLZxqV+EbvgroiiGEWD5FxDU925aqk+efmZkJPv3pz18/MdG34U//9CMvTP4+qT5fu/b/mAEYGvqnMwcOnDzvE5/4xDNTY5B1HOqbjL8IfBj4NHAQeBHwT+PjZcDZK53LKqmchTsVjJlAcZMQnUYV6EIIkS+JgJ7ldb4lQxJ8nLuvCTAhOkHZE6ldTQZtfMfGvcDnj33w2Jv3vnrvb9gZu8Zaa7Z9bNtnwrFwIOl5vu2T277CIubA3ztwW9ITffhlw8ezjNGCceDe+NhIvTL9zbgebUllepLMqJKAnnslRUypKtCFECJFlmekz2toXzcP+Bgrgp9zrjJ5XM+q3yM9IaD/5//8ny8/cWJ2UxD8RvijH91+5S9+8YsfX3TRRdNf+co911p7nYmig0EUHQwAFhef15eqQl/N9UnE9MeAGvAsXMx0HXADrho9qUwvUkAvyhI89zmFYVhZAV0I0VkkoAshRL4kC8x2i2dZuPuJb/P2bb5i5SwVMFehAj1v2n7+je/YuHfx8OLXTvz3E7cCbPuf2z4+dO3Q2PQPpjcPPHPg4Hn/7bxvJrbtF9x+wZcO/9bhG4O1wWL89k4lNMaAH8ZHWkz/NerJjEuAHcBCsxN0gKIF9FzHWVxcTOajZJAQooyEZIuZwE/nLp83D/g257JvFhXdpwoV6HnT9vPPzMwEn/vcl6+DNzA4+MG5ubmvDf7pn37khW9/+9vun52dGezv/9uFhYW/7U9ePzDA/PHjC5seeeSRdZdddlmn5hDi2l49itNxGsX0EJgDtuD6p+dFpSzcq1yBLoToLBLQhRAiX2Th3h4fBXQfE2BKBFWXTl/bqn4virJwb3v+uUfnhsf/ZvzFyf8f/5PjN6y5Zs0X1v7q2hNrf3XtXenX9m3tm7/wby68s2GMTpMW0zfhev5dBbwWuJl6ZfpjdFZMr5SFycqgSgAAIABJREFU+8LCggR0IUSZyRozkeF1VcTXufsYK4rq0sm1YFa3wzLSEwJ9Un0+MPC+BWPWEgT/LPrRj/7tle9//+/8+DOf+avbZ2dna43v2bBhw8Kzn/3sKfKZwyLwSHwkYvobgW3APweO4mKmnXReTC+yAj33uCkMQ1DMJITIgAR0IYTIlwhZuLfDx7lXNdAV/tHp726W85W5R3netE3UJD3P7YLtv/AzF/7l3GNzI8c+eOzNB9564I0XfPaCL9TW19olEvJOaI0CD+ME9L8EnomrsngLTjx/FJcY2sXqxfRk41olKtAXFhaS+SgZJIQoI1niJl9FZPB37mVel60YD636xfLp5j1SZDVyHrT9/Onq8yB4TgTQ3/+uhbm5/zb4p3/6kRd+6lP/6xurHWOVJGL6Sdwm4724mOl64BXAEZyQ/iBwogPjBfGYRaEe6EKInkACuhBC5EumCnRjTORxkOyjgJ7g27x9m6/oPGW+h4qqQG/JxP838Sy7YPu3fWzbZ4auHRobunZoDPj8sT88dtvUPVNbN7x+w6EMYxTVm3wM+H58bKZu8/5WnHj+CHUxfSUJnWScIqopcq/aUAW6EKLktP3dYozx1sLdGOOrgA7+zbnMwqRoTtH3cNk3BhdRgd7y/J/85CefOT5+cgTuNHNzFw8mf2/trPnxjw9vf+qpp+7ZsmVLq828RfYmD3EbkB/GaT2XcKaYnvRMH13FOEXNB3KOm9QDXQiRFQnoQgiRL1kTHd5auMfJIF+TIj7Nu6zBu/CHonqg5/1daHn+s3/v7B0b/8HGR/rP7Z9L/m7jOzbuHX7F8F+k/64FRQro6XFOUBfTt1AX098GzHN6ZXpWMb1SFu5JD3RrrZ63QogyktW1y9fqXB/jB/B3s7VPc/ZprlWh6wJ3B2h5/ltuueXAkSNHvr64uHhGnm7z5s0zIyMj7eKNIgXn9DiLwEPx0Q9cjIuZXgq8EjhM3eZ9OWJ6URbuhTiExQJ6UZb0QogSIwFdCCHyRRbu7fFu7sZ4Nd003k684hQp1pXdLjBvMv18mgnlGcXzzGOsknbC9lPA9+JjCy4pdBlwOU5MTyrTd9NaTC9SQC/Kwl2VFEKIsiIL99b4unlA+8JElSjqZi57BXretF2XX3jhhbO///u//8s8x+gQrcZZ4HQxPalMvwG4EThE3eZ9bBXjdJJC7l1ZuAshsiIBXQgh8iWThTsuYeRbMiTBOwE9ha/zFtUhj3s4eW4WTRUq0ItKbHRbQE/zFPC38XEW9cr07Tgx/WHqYnpjkqRoAT3XKofFxUWTsjcWQoiykTVmAj+du3zePODbnCV6iqx0614pewV6Uc+UbgvoaRZwYvlOnJj+bM4U0xOb9/FVjLNaConPZOEuhMiKBHQhhMiXJMkjC/el8XXuvm0cUCKoouTgqJDleZknZb5Xi3im9EIF+lIcpy6mn01dTL8CmOP0yvSQgiwCYwqpQJeALoQoMapAb42vc/ctZkrwcc6VpaLOEWWOyYqKZ3o1xligLpYPUK9MfznwKuAgdZv3REzPvR1VTCE90FWBLoTIigR0IYTIl6wV6LaiQVUWfE2KlFmkWyk+XmexPNrdI3neQ3nfn1USt3tVQE9zDPhufJxN3eY9EdMfBp7swDhZKaoHuhJBQoiy0vYZaYzxVUQGvwV03/BxzmL5tHoWFOWslRdFnL8KMROsPsaY53QxPalMfyVwE3AAJ6T3rXKcrBSywTmKIrW+EkJkQgK6EELkS1YrYq8t3H3dPODTvFMJT1E9On1tu32v5Jloqoq9ehkE9DTHgO/ExznUK9OvjP/95cDPgT3kl0jJ3cJdFehCiJKTNWYCf92rwM+Y0cc5i2rS7Tink5S9QryoeCZvOvmzmgceiI9BThfTa8AG4EU4QX2iQ2M2UlQPdJCALoTIgAR0IYTIF1m4t8fnCnQf5+0TPlzfPObYrpqirJUUUB1xu2wCepon4+M7wFXAbcB5wDuBWeo90x+ns4J37j83VaALIUpOlmeuzyKyr3P3NWbycc6ic5S9QhzK3wO9ly3cszAH3B8fg8B747FuBG4G9lO3eZ/s4LiycBdC9BQS0IUQIl8yW7hneE1V8XXuVdp9nhUfr7MoH2UW6Ks2Rt7PybH4z0/iLAsTm/ergBnqYvoeVp/EKcLCXRXoQogyk+UZacEvF6cEj+3rfYyZfJxz1enW9zbPe6nMAndVNh0XNc4cbqPxQeCbwHNwMdOrgFcD+3BCeifEdFm4CyF6CgnoQgiRLxEZBOI44e1bMiTBZwHdp3krESSy0q71RVkF7qokaspegb7UOEfj49vAVupi+tU4Mf0hnJj+BCsT03O3cA/DUJUUQogy0/YZmdok5KNzl88Cum9zBn/mrBhxdXTr51f2uKkqMVM3xpkDdsTHGupi+k3UxfSkMv3kCseBnOekuEkIkRUJ6EIIkS9ZEx2ycPcT3+bt23x9oZPBraG9hXvelLVSo6gxqiSgL1XhkBbTz6Uupl/DysX0oizclYgWQpSVzBXo+Lmm9LL63hgDfl5vUU06HTet5N86Rdl7lFchZipynGZuWrPAffGRFtNvBl4D7MXFTA+RXUwv0sJdzl1CiLZIQBdCiHyRhXt7fJ27RI5q48v1rdJ3t6d6BYZjYV9tY22x3d81GaMKyaCi7qssQv2R+PgWrlf6ZcClODF9mrqYvpfWSZiiLNxVSSGEKCsS0Fvj69x9WVOn8XHOolz0VNwEEEURd95553k33njj0TVr1rQTRtUDvfPjNIrpz8XFTa8GbsFtPE7E9KkW55GFuxCip/C12lEIIQph/fr1lmy7GmXh7h++zVuJINEJylxhm/mzT31vasvem/e+a+LzExckfze/e37tvtv2vf34h45f3okxVkFVxkjGgezVB4dxff/+HPgY8AvgWcC7gPcDtwJPp/mzvagKdCWChBBlJWvMBH7msnwW0H2bM/g55yqTx/Vsta4sewX6stfNn/3sZ5/2r//1v/u1P/7jP74yj/OvgF4RtrsxzizwS+Cvgf8EfBkXo7wGFzP9PeD5wPAS47CMsVZEXIGuuEkI0RZVoAshRP5kEcdl4e4nvs5biF6kZ76Pa65aM9Z3ft/xJ//gydcCX11z1ZoTB//BwTfZBds3fOPw/hZvLWoOVUo4scKxDlMX1M+nbvP+fFxVRboyPfk9l6tNYFyBLitCIURZUQV6a3yeu29zLutmUVEgGdo55C1w9wxRFPHxj//1CycnN6z/6lfvufa3fuu3HtiyZctCi7cUFWv0mrC92nFWEmfM4DYd/wIYol6ZfgtnVqZPU6yFuwR0IURbJKALIUT+yMK9Nb7O3bd5J30bk16Gojp0JGC31pooitbiAutW58wrQZB38iHz+Wvra+G2T2674+C7Dt765O8/+QaAYENw8rz/et4Xhq4ZGm/z9iqI20X1pOtUhcOh+Lgb2Ebd5j0R03cCGzowTktUgS6EKDltn5GpNaSPi0lfBXTfYiZRbTqyFoyiaMBaOwzUOnG+FdITcRO46vM9e566cO3aP54eG/v9kT//8z+//IMf/OAv2rytCjFTMk5RcdNq55MW09dSF9NvjY89uM3HdGCslsQbUBQ3CSHaIgFdCCHyRxbuLTDG2Ay7p6uIKguEV1hriaJoXRRFG6Mo2mit3WitPfXfwEbqSaBLgB3A/cDJ1GnK/KxYVtKhtr4WnvPBc75z4B0H3gWw4U0bfppBPK+KHWEZKtCX4mB8fIO6mH4ZTkCPcNaFDwL7OzyueqALIcpOVgEgstb66Nzls4DuI75dZ5HCWtsfhuHGxrgpOXDiI8AC8AAubkpcj6CYjcF5k+nzJ9XnYfj82tDQm2cWFx/s++pXP9OuCr1q8UwZx5kGfh4fa4Hn4WKmG+J/vyn+t4dwwntHUQ90IURWJKALIUT+yMK9NT5XFfg47zL3sBZncuoebiWQAxuttSOcvvacNcaMGWNGgyB41BgzBpjFxcVzgQHglcCrcDvR7wMeLmAuPXNvzu+eX3v4/zz8erPWzAZrgrmxvx67fuAZA+Mb3rzhQIu3VSUZVNS1SH7v5jVWIqbfDfw6cC6u0uIFwCSuMn0nHRLTwzBUJYUQosxkfQ56GTsYY3wV0MG/OffMelR0jNPu4VggH0lipvjPTU0EcnD5pPE4bjpaq9UeMcaMLS4ubrPW9gNPB64GJnBC+o6C5tQT92lSfT44+J/mjAkYGvpHM2Njn21XhV7EM6Wo51aRcVNe40wDfxcfFwHvxs3rtfHxOG4D8sN0SEyXhbsQIisS0IUQIn+yCui+JQYSfJ27b/PuiQBbrBxrLdba4aQaIk72PN1au3FqauqfxcmeZgL5mDHmMWPMWBAE6WOucYzFxcXzFhcXA+BeYA3OBvsK4I24CguLq1QPKMaqrpNkTm7M755fm/Q8P/9/nP+5gUsGTh5818Fbk57oLUT0opJBVRHQ86hAb4YFZoETwO3ABdRt3l9IXUx/EDiw0s8Tf3fK9r0QQoiErM8v39bQCb4K6L5ebx/nXBmstX3pmCkMw60A8/Pzr5mfn18DDKde3lQgT8VMk6kNNKcIw3CNtfYw8GXgQlzM9HzgJTgx3QDrON3Nq1PkeX9mXp8n1ecLCxf3Dwxsnl9c3FkDsPalpk0v9KrFM1UaJ8kR3AGMU69Mfx1niumzKx0kdrKRgC6EaIsEdCGEyJ+2AroxxlsLd/xNikhQFj1FM4G8iV1geu04hws6bRAEuxoSPeNBEKw4oI2ZpW7rNoJLCt0AXI6rtEjsCg+vcpyEIpICmc5v520QbAimzvngOd9ObNu3fXLbHYf+4aFXR3NRu16HVXi2FC2gF9k30OIqzvcDX6cupl+GE9MnqIvpB1nGzyGupJCALoQoK1mfdxF+OndZONW31SeqsK4RrSndPR0L5OkK8k0NMVNaILe49R3GmLFarbY/jptGWwnkWT5G6r+TteXXcK2wbsC1D/odnOC4A2eFvZSl+Uroety0e/fu4YMHD53b3//Uwvz8bQPJ3/f1sTgzs7Dm7rvvPu/tb3/7viZvrZKATkHjdGOD8xTws/hYR11Mvw0nqK9YTFfcJITIigR0IYTInyyLMgsYay3GlC5+XC2+Cujg57x7yibbJ2KBfG0Yhqf1Hm9I9vSn3jKXVJAHQbC7SQX57MzMzGuiKLpoaGjo6zl//HHge8CvALtxyZ/twIuAY9T7pbfrEd5NMn/fB583ePKir1z0hfTf1dbXwgs/e+EdGd5eRL/DqiScippPMlbjeqBRTL8QlxS6HHdvT+CSQjtxlektWVxclBWhEKLMyMK9Nb5WoIN/c1as1GWaCOSNMdO69MupV5Afq9Vqjc5bk1EUrZ+ZmXlff3//DwcGBh7v0Mds9r0IcWLiBlwP6buou3m9Fiei78AJj6u5z4qoQG/LJZdcMvXJT/7lJ0dHRwca/21wcDC67rrrnmrx9irEM92OZfIaB86c00ngp/HRTEzfjYubHiGDmK4KdCFEViSgCyFE/mS1cAc/xUWfk2A+zdu3+7pwViCQzxtjRuMEz+PNBPIMw+ZxD7e6VwwueL4H+AbwTFxS6GW4nulP4JJCO6nbv2Ul7+dvEeenAmMk41RRQG81jgX2xcfXgKfhkkLbgRfjNoekK9PPIAzDwBijRJAQoqzIwr01vgrovl5vH+dcGNbaWkog35RBIJ+IBfKnarVao/PWZOwo2Iq8rudSa8tk3Zn0ld6IW1NeiYudTuI2H+8Ajqxg3DzX6stan19zzTUr2UBdpbZXFDBOMlavOISlxfT11MX0N8Tv20VdTG+aE4iiSAK6ECITEtCFECJ/sgjoyeJQAro/+HadE3y81h0hFsiHoijamBypRM+mJQTypIJ8T0OiZ9QYM1tyx4skON4FDOAC5yuA1wO34gLm+3C70XslOC5C3M4TCegrJ1jGOBbYGx93URfTr6Aupj8YH4eSN4VhqAp0IUSZyRoTeG3hjn9raV9jJrEKrLW1KIpGltpYjBPdTr2c0wXyRuetiQwCea8zhnPz+h5wPm5NeSVuXXkMFzPdT2w13yOUeWNzkWNQwDjJWEXFTCxjrEngJ/GxHrgUFze9ERcXpSvTT4npcTsUxU1CiLZIQBdCiPzJsihLFocB/vXh8VVAB3/nLZrQRiBPkj1pe7qFVAV5o0A+ZoyZKUgg71ggnbGvZ7Px5nGJn/twgfN2XGLo14EZXL/0+1iicjemqArxvM+vCvTsLDdBsxpWanvYKKZfhEsKXQVcB4ydPHnyoXvvvffIwsLCDEoECSHKS0S257GXsUOqR7J3c8e/OWvTQBtWIJBPxgL5iSAIHm9w3SpMIF9hr/NOcyg+vgE8Cxcz3QDcSHY3ryIq0PNEAvryWc5m4NWwmjlNAj+Ojw3UK9MTMX3XD37wgwMXX3zxfapAF0JkRQK6EELkT5Ykj88JEZtRNKsavs27F5IFXSUWyNc0COSb2gjkSQX5XmPMfUn1eMECedG0s3BvxyTww/jYiksKbQeuBU7gkkI7gNHVfcxlU5QoXIUxqliB3ok5WVxi8wlSYvo999xzzXvf+97BCy64wJ511lnHjTHXAL+w1nr/3BVClAr1QG+Nr/Gir9fbxzmfIhbINyy1sRgnjqVJKshHgyDY00Qg900oy7LujIDH4mOQ0928bsFV7O7AuX0VWeRR1Pq8KjFTEeMkY/WKhXsWJjhdTL80iqLLPvCBD7zyyJEjr7zkkkvCBx544EZjzEPW2slVjiWEqDAS0IUQIn8yW7hba01FBbFW+JoU8VXYqPS1jqJoTRRFG4EgDMOLZmZmRhqSPYOplzcVyI0xY7VabcwYM12C50EeHzDrhqMsHAXuBr4JPB1nVXgdrspiPy4p9CCuSr2IpEDZq8OrVoFeZNuUAFjs4PkiYA+w56abbrrrox/96Au++tWvvujuu+/ehOt1udsY8zngr621D3RwXCGEyIvMFu7WWlm4+4OPMVPl52ytDaIoGomiaBPQNzMz8/ImFeTpe30iFTc9UUKBvBsx03KYA34ZHxs43c1rGufmtYO6m1dV4qY8qVoFepkdwiaAHwVB8KOPfexjF3zlK195yWc/+9mLgT8E/m9jzF3AZ4EvWGtbOS8IITxEAroQQuRP252TKSsvX5NBviWCEnyadyUSQVEUDaYqIU6rHm8UyK21l0RRdCJO9Ow3xuwooUDea6z0B2aJxUbgDuA5ODH9FuA1wKO0tinsBLJwz06RFoFFPZtyG2twcDB885vffO++ffvGvvOd75w7Nzf3B8BbgX+EW4P8m5We2xhzJS65tB04G7fZ5BHgv1lr/3r1n14IIU6RtdrM19jBVwEd/JxzqYkF8nYV5KeuaxiGV6c3FqdjpiAIxksgkPcaq1l3TgA/iI+tuJhpO/AC4CmckJ4nVRK3qyagF+FEkOuctm/ffmD79u2f+fjHP/5PgHfh7u23An8BfHE151bcJEQ1kYAuhBD5EyIL91b4nATzcd49TYNA3qwH+ZrUyxeNMWNAIpDfn/Qhn5ub+81arfa3Q0NDP+zSVIqk08Ft3gmABVwVxQPAMHA5rsLi/PjfX4tLDO3v8Gcpe4/19BhVSDgVOU4yVq5JpyiKjDFm0Vp7N3C3MeafcvozayVsBPYBn8ZVHQ0D7wT+lzHmImvtH63y/EIIkSAL99b4Gi/6er17es6NAnljH3IaBHLqPcjHgiDYlwjkYRheHIbhFevWrftIl6ZSNJ1cd9oOn68ZR3G90u8GnoGLmV4S/9t18Z87cUJhpyhqU3BR4nYRY1QpbuqUhXtL4h7o+621Xwf+kzFms7V2dpWnVdwkRAWRgC6EEPmT2cI9w+uqiK9JkUpUZK+Arl7rWCAfiaJoUwaBPEwJ5AeMMQ8kAnlcQT61VAX53Nwcqi5fEVl+aJ387kxR7432KuCFwMXArwBjwP3Afbhqi7JQhQr0KgrouVfVh2EY4DbtAWCtXcBtGFkx1trvAt9t+Os7jTHPxFW4KxEkhOgUEfW4oNXzMsJf1y6stV4tMFNObT7R9TnHAvn6JgL5piUE8pMpgXx/IpDHcdO4MaZpG5vZ2dlthUyomhT5LLDA4/FxB/B7uNZEt+LcvB7DxUyPkVqL9jhViGeqKKDnYeF+BvHv0ij1/yc6cE7FTUJUEAnoQgiRP1kWfklCxLtkUJwU8SoRFOPbvAtJBEVRNNCmgnwo9fIQSJI7B40xDyaJnviYWkXSruuJr4Io+h7Oc7xpnNj4Z8DTcBUWLwCuBw7hqtIfwInuK6GoCvSyV7kn4xSVoCm9hXtCGIaG4pKWx4FzChpLCOEHmZ6RvsYOqTWpb3P38nqT85yttaZNBfkIHRDIxSm6ETPlte5MrvWPcIJ54ub1NmAWeBAXN+1b4fmrEtNUTUAvssUWeY8V514VNwkh2iIBXQgh8kcW7q3xNSkCHs7bWmtWU5m9AoF83BgzGgTBIWPMzg4K5KJ75H3NLLA3Pu4CLsH1/rsJuBnYhUsKPcLyq3urIqBXoWKjyHGSsXK1ImysQO8k8YM7ADbh+gTeDPxWHmMJIbwlsSNu92z2NXbwOV70cc6rIhbIGyvINzVYrKc370+lBPKDKYF8VAK5aMFJnJD+I1zP5ytw/Z8TN68d8bESN68qWLhXSUCvlIV7ngK64iYhqoUEdCGEyB9ZuLfG9ySYSGGt7Q/DsJVAvjb18kQgHzPGHK7Vag81COQnJZAXQpE/4zyfFc2SAovAQ/ExBFyGE9PfDMzjev7tAJ5o8t5m58+TKvXZKzJBk2tyJkURFu55VqD/d+C98X8vAv/SWvs/chpLCOEnaQv3dq/zzrULfwV0H9fy1trW015CIG+sIM8ikCcV5Ktq+SJak7qeRW7czPvcjXM5BnwL+DbOzetKXHusl+L6QSduXtMZz58nEtBXRpUs3POsQFfcJESFkIAuhBD5k0VATxaHviaDfEsE+Wg/mbQp6F9cXBzJKJBHxBbrxpgjtVrtYQnkoovMAD+Lj024CosrgauASeoVFk8u8f4qWbhXIeFU5DiFjBVXoOdVIfaHwF/i7AdfB3zEGDNorf1QTuMJIfwjq0Ds2xo6wWsB3VrLalysyoa11oRhmBbIN7URyKdTAvnhpHpcArnXdCtOTrt53Qk8Gxcz3Qy8GufmdR/OzavZurWomKYK8UyRsRkUs/G4ChbuipuEqBAS0IUQIn/aLjKNMapA95PKzTuuIB9prISIoug8gJmZmQ+kXh5RryA/WqvVHmkQyCclkPc21tpu3MN53RPLSXKMAt+NjwtwYvo1wK8CR3FC+v04YT1NEfezBPTeGycZK9ekUxRFxlqbi4BurT0AHIj/92uxNeF/MMb8T2vt8TzGFEJ4R2Lh3o4sm5OriK8CekKRv7NzJ64gX9esghxYF0XR9TMzMy9NvaWZQJ5UkI9JIO958vreduM7sZx4YBHn2LUTt1H+Mlzc9Gu0d/Mqom1XnlRRQJeFe7ZzK24SokJIQBdCiPzJXIHeJTGq2/gqoJdy3tbavsRivdEqME74DKdenhbIx621m/v6+r5Sq9WOSyAXS9DuO9GL35kkQP46cDEuKfQK4FXA47ik0EMUZ+FedpEeqiugF1GBnlclRSM/wVkTPhNQIkgI0QmyWrjbOPHtG74K6Ol5lyZuiAXy4SiKNrWwWK+l3jKTCOTW2gVjzP7+/v6fxDHTeBAE812aiuhdWj0LevH7Mg38ND4242KmK3BuXhO4zcc7gKkCPktRz9GqCNvdENDLXIHeiOImIUqMBHQhhMifEFm4t6KUQnIH6LWAFjglkDdWkG9aQiC31AXyY7Va7bFUNcRoWiCfn5+/dH5+/hkDAwMPBkEw15XJibzo9L3cre/GahNNIc6K8BFgDXApLin0BuBWYBy39g7IZ0e9eqAvn9z7khc5VhiGuVWgN+EVuPt4d0HjCSGqjyzcW5CyL/dt7r0aM9FYQd5QRb6RJQTyIAgaXbfG0gL5yZMn3xcEweGBgYHHCp+YyJuevJ+XSSfigRPAd+LjQk5380raYQ2u4vztkIX78iikL3nBYxUpoCtuEqLESEAXQoj8ySKUyMLdTwqf9xICeboaYl365SwtkCcV5MsVAn291qIz5J2I6NS5Z4Gfx8cIsB24DhgCfod6hcXhDo0H6oHey+MkY+Vu4U6HE0HGmL/AVQX9BNee4CzgrcBbgA9Za5/q5HhCCK9JKtCzvM7X9aTipoJoJZADG621I5yeU52NY6bRIAgebYiZxpe5gbgKIuty8PWezpMifqaduk/3x8fXcG5e1+B6R78BJ6zvAB7GWb53CgnovTlOYWPlUYGuuEmIaiIBXQgh8iezhXuG11URXxNBuczbWltLCeSbMgjkE3Gy56larbY7TvaMrkIgX/Kjdeg8oreo0nc3r7mMA9/HuTc8F5cA2g68CGfhtiM+xjs0ngT05Y2Tq6jdMFauc4qiKLDWdrqS4ofA3wfehaumOwncB/yGtfb/7fBYQgi/SXqgZ4mbfHTtAj/jplziZGst1trhdGuqBoF8I80F8jFjTOOm4rEcHLZ8u85VpxvXM691Z15zSdy8DuE2HP8EOA94I7CAa4m1A9hDZ9bvRYnOeVJUtXZVe6B3egzFTUJUEAnoQgiRP8sR0H1MBvmYCIIVBh/W2loURSNhGC5VQb6+YYxTAnkQBLsbEj0THRTIhSiKMlSgN8Pgkj9fB+7G9UC7AngpztZtLy4ptBNXwb6S84ME9F4cBwqycKfDlRTW2tuB2zt5TiGEWAJZuLfHx/7vK42ZmgrkDXFTOic6Z4wZjeOmXU0qyFeyNhMib7q5ST1vgX4Xrip9I27z8ZW42Okk8ABOmDyyyjHypIoV6EXkjnLfFBBFEbg5KW4SQrRFAroQQuRP20VZSsT0MRnkcxLsjHnHAvmGMAw3ZRDIoS6QjwZB8HgJBHJfr3WVKTJxk+f9U+T/P1czAAAgAElEQVS9GeGSQruAAeB5uITQa4FbcNUXO+J/zxrYFymg500VBfQiLNyDAnugCyFEp5GFe3sUN8XEAvnaJTYVb1pCIE96kO9uUkHeSwK5nLuqS5HrzrJVoC/FGPC9+DgfFzMlbl7HcDHT/SzPzUsW7r05Tnqs3Jifn09E+qJ6oAshSowEdCGEyJ/MFejWWu8SIsYYrxJBiUBure2PoujsmZmZlzcI5Bsa3pIWyPc0EcjLsuhXIkhkodvPgrzF52bnn8dVUNyH2yCzHZcYejswg6uw2AEcyHD+vFEFeg+PlUcFuhBCFIgs3NvjTdyUCOSxpTqzs7MvAtY3xE39qbfMJxXkJRDIs+DFdRarotv3SN4CfbPzH4qPbwDPwsVMLwNeCTxB3c2rXUsFCei9OU4yVq6bjhcWFpL5KG4SQrRFAroQQuSPLNxbU6lEkLU2iKJoJIqijcnRpILcxK/dEobh5pRA/kSJBfKsVOZaCyCf69kqMC9zNUWWzz6J6532Q+AcnFXhduBa4AT1fumjLc4hC/fs5G6rXuRYOfVAF0KIokgq0GXhvjSVmXsskA81xEyblhLIwzB8SaqCfE+DQD5qjJk1phI/GlFNir45y+zalSXeiIDH4mOQupvX6znTzWspMbaIeCZvN8CiYrOieq0nY+U6zsLCgirQhRCZkYAuhBD5k1RTtEIW7iUhFsg3tBDIN3D6fCZTyZ69QRCMGWPG5ufnbzLGHBoeHv5yl6YiRK/SzedBLzklPInrlf5N4Om4pNB1wA24avT7gAdxVeqgHui9PE4hY4VhaGThLoQoMZmekXF7otLEDh2mNHHTEgJ5Y9w0kHrLQqqC/JRAHobh5jAMbxwaGvpIrVZrV1VaFXppPSp6EGsz3SJ530e9cp/OAb+Mjw3A5bhNyL8OTOPipfuAg6n3qAd6b46TjFWEaxfW2l5rdyiE6EEkoAshRP5kWfwlrylFQqTDJPb19ELVQKNA3thTj9YC+b5EIA+CYKxWq40vVUG+sLCw2AvzLZBeCbCLxJcLXPS1rXIFejMssCc+7gSegxPTbwFeg6u82IHrAZi8Pi+qKKAXlThRD3QhhGiNLNzb0zMCeiyQr0kJ5JsyCOSnNhUbY+5Lqsfj+GmmWWw0Pz9/aRiGGM8CJ3rkOouOU4WYuBcq0Jdigrqb11bq/dIb3bxk4b6ycYqIm4qycFf1uRAiExLQhRAif9ou/uI+4OBnMii9eSD3gDIWyNcvIZBv4kyB/GQq2bO/iUC+UrGiCsHzSlAySLSjnYV7t8buhfMv4HqiPwAM4yosrgDeguulDi5ZtK8DY7WiCgmnIseBYizcjSzchRAlZjkW7j7GTFCwgB5F0Zo2FeSDqZc3FciNMWO1Wm3MGDO9Qg3cx5jJxzmLlbHUvVLEGrfXz3+UupvXM3Ax06/i3LxmgXFgiLqbVx5UTUCvjIV77GYjhBBtkYAuhBD5k2Xx57uFO3QoyLPWmjYV5CMsLZAf6KBA3vaj4tf1ViKomnT6Hu7md6KoaopOMQX8OD7OAl4IPB9XlX4d9QqL4x0cs8gkTVEVDpWxcI+iqMiKeiGE6DRZHbkia20t7w/To3Q0foiiaDCpHs8gkC+mNxUbY+6Pe4+vViBvh89ObT7gS4yomKl757fA4/FxB/Bc4GbcpuMPAI/iYqZH6WxVcpEV6FXptZ6Mles4i4uLRcaAQoiSIwFdCCHyJ7OFu7XWx8TAspIisUDerIJ8U8piPV2VMpVK9hxsEMjHchTIs+Dj9fYFBWT5UdbK6jwD9eM4If35uMTQuTirwuuBw7i+fw/gRPfVUEUL9yIF9FyTW2EYguwIhRDlJbFwz/I6VaBnICWQL1VBvib18kVjzBhwSiBviJmmuuyi7k3cFP+cvZmvKC15t9XK4/wLwP04W3dwovoVwFtxVekP4sT0/R0Yv4oW7pVosRVXoCtmEkJkQgK6EELkT5bFnyzc40V5C4E8XUGeVSAfN8YsFD2hjHgpsHq6SaTqFHkv53n/FHFvFtGf/AjwM+BrwCW4pNBNuEqL3Tgx/RFcAmmlY1RFQM/dIrDIscIwVD8/IUSZWY6Fu6/rydPmvkyBPEwJ5AeMMQ/0mEC+FD7GTD7O2Req4HxU1IMi77hpEfhRfJyFi5muAH4FGKPu5vXUKsbIm6JisyT/Von4LK5Al2uXECITEtCFECJ/slRTeGPhHgvk65JETxiGzwGYnp5+p7V2hDMF8umUQH44SfQEQTDa4wJ5O3xL/ikRJDpFnsmgIgTuos6/CDwUH0PAZbik0Jtx/dIfwiWF9pB93kU9s4qsQC8qeZL7nCSgCyFKTmYL9wyvqQRRFA2kBXJgIAzDy6empp4TC+RDqZeHwKlNxcaYB1Nx01gQBFPGmDKux321cPdtvqJ8lFmgb1yXHwe+DdwDPA0XM70QeClwCBczLdfNSxXoKx+riB7oipmEEJmQgC6EEPnTNjmfSmaUPlBuFMiXqCBP9y2cBzDGzAZBcCid6ImrIcoqkGeh9Nd7Bfg45yqTx/VsFTCX+f4pSqBvNsYMrir9Z8Am6hUWVwKTOCvDHcDRVYzRSWThvgLiHuhKBgkhykpSgd6Oyli4NwrkTSrIGwVyY62t1Wq1vcaYnRURyDMhFytRcoq+f8tcgV5EvLHUz8cCe+PjLuDZnO7mtQsXM2Vx86qigF7ExuNCeqAbY1SBLoTIhAR0IYTIn8w90ClBMigWyIfjRM+mDAL5TKqC/BFjzGgq0TO+uLh48fz8/FvWrFnzxSAI5ro0rW7gWwW6EL1OEUmOIhLb7cYYBb4bH9twIvrVwHU4AX0HTlCfbPLeIpM0lUjQFDmWKtCFECUnce2qjIV7LJCPLCGQb+J0gTwiriA3xhyu1WoPNQjkJ6empn67Vqs9MDQ09O0uTakbVGaj+TKo7EYI0XG6ea+UPW5qd/5FYGd8rOVMN6+duLjpiSXOVUUBvRIOYYqZhBDLQQK6EELkT5ZFZs9YuFtraawgT4vkwEZaC+TpRM9YEATz7YaM/+z63AvGt8SIb/P1iU5e2yzPgbLeS71YrXEwPr4OPAsnpr8CeBXwOC4p9BCxU8gKx1gJVaxAz72fnyrQhRAlp3QW7tba/jAMW1WQr029PC2QH6nVag83CuQZKshLs3mgg5R13bdafLvOvtCp+7mb90fRbal6YYxp4KfxsZm6m9dVuE3HSb/0JxvGqIqwXaSAXkQP9AD1QBdCZEQCuhBC5E+WhVlhInIrgRzYGPchT/9+mE0J5I82JHrGO1A17quADpqzEM3oloV7ERbr3bJwb0cIPBofg8CluKTQG4DXAg8D91G3KqyKgJ57giaFqimEEKI1PWfhHgvkIw1tqTa1EMjH47jpaK1Wa9xYPNkBi3UfBfQEn+bt66YB0TmKsHAv6/mTMVZ6/hPAd+LjAlzMdA3wqzg3r/tw/dKrJKAnv3MrscE5tnBXzCSEyIQEdCGEyJ/MFu7W2lUng6y1WGuHk2qIJgL5RpYQyI0xjzWpIM/bVt1XAd23BJgSQdUkj3s4i3WrOJNOXYs54BfxMQJsx1WmXw7Mxq85CzjeofGaUcUK9NzHUgW6EKLkZLJwN8bYTvXDttb2pWOmxipyYDj18iIE8rYfGb/iB/A3VvQJH66tD3MsE51alx+Ij68DF+PE9Ffi3LzmcZuPB6i7eXWaKlag577pOK5AV8wkhMiEBHQhhMiftos/Y0xmC/elBPKGZE/6+T4XJ3pGjTG7mlSQzy41VkH4mhSRCCjE8lEFens6OcY48P34OA9XWXEZ8DacgJ7YFY53cEwoVkAvqtc6FCOgy45QCFFWkgr0LBbumTYdxwJ5UkG+qY1AbqkL5MdqtVrjxuKJAgTylsTjK2byA9+us+gsqkBvTyfPHwKPxMcanJvXzcDTgQ/g3Lx24FpkdXKtXkUBvQgLd9Pt3+dCiPIgAV0IIfIncwU6YGKBfG0Yhs0qITY1EcjnY3F8LAiC3U0qyLstkLcjqb73MUmgOQtxJlUNZnvZwj0Lh4Gf4QT0L+J6pl+P65m+F5cU2km9Sn01VK0CvUgBXdUUQoiykvUZeUpEbhDIm1WQr2t4XyuBfDK1qblX8VFAT/Bp3lVdC4vOU8V7pSgL97yYBX4OPB84htt0fAXO1WsKuB8XNx3uwFhFC+hFbTwuoge6YiYhRCYkoAshRP6ctvhrJpDH9uosLCzcuLCw8FqgP/WWlgK5MWbWmFLnE7ysQPewgqSKwb1wFH1t86x2KPN9WmSyaS+uv98A8FycxftrgVtwlRc7gF2sPDFRVG/yogV09UAXQoilOcPC3VpbiwXyTUncFIbhs4CRkydPvp8zBfKJWCB/qlarJXHTaIkE8nb4Fj+QqhL0at74N18vqEjVa5HVyHlRVH/yOeB78XEeLmbaDrwIJ6zfhxPUV+rmVcUK9NwdtRQzCSGWgwR0IYTIn2h2dvayKIquSFWQNwrkYwDGmOO1Wu2xBoF8puQCeTt8TYqUOeAUIqHT39t25yvzc6KoCvQ8aUyezFO3cV+P65N+BfB2YAZ4EJcYOrCCcaokoCdWw6pAF0KIpYkWFha2LiwsXGqtHYpjpvWpf7fABO7ZHdZqtb9rcN2aqIBA3g7vBHQUM4lq0I2YqazfnaI2BRclOiccjo9vAM/ExUwvw/VMTzYn78SJ7ssdo0oCehEW7oExRjGTECITEtCFECJ/rLV2GJgNgmBPkwryGWMMJ0+e/P2+vr4HBgcHf97tD1wwvgrooDkL0Yx2AXOZK9DL1gO9kVbf30ng3vg4B5cUugJnXzhKXWg/kXGcohI0VeqBLjtCIUSZsUCftXa9MeZYEAR7UtXjpwTymZmZV4Vh+NyhoaHvdPsDdwEfBfQEn+ZdVuFT+EEZe5R34/xLxTMRzqlrF87N63m4mOn1wK04N6/7gN20X9cXdS0K2QwcU4SFuzYdCyEyIwFdCCHyJxoaGvoJrhKvFZb6wtQbPLbl8yoBVhG7OlEMVf1e5D2vIi3c243xJPBN4FvA03FJoRfjqiwO4IT0B4HpFuNUIkGTGgdyFutVgS6EKDlRf3///v7+/v243yNL4dUaugEf524BrLW+zdu3+YrOUub7pyhXrW4J6GnmcWL5fTjHle24uOnXcW5eD+DipqXcvKpYgZ67hXscM1XdsUYI0SEkoAshRP5kXWRGHiYGwN8KdB8TYODnnKtOVTZHFGGx3gvidpFjWGBPfNwJPAeXFHoN8GrgMVxS6FFgsWGcSiRoYmThLoQQ7Ul6oLfbUBxleE1V8TF+qMo6czn4OGdfKPLa5unaVebzp8fIk+XGM5PAD+NjKy5m2g5ci3PwSty8RhvGYJnjrIRKWbiHYSjXLiFEZiSgCyFE/ixHaPAtIQL+VhWAn9fbJ3y4vnnMsVsW7kXQS+J20WMs4KooHgCGgcuAK4G34Hr97cRVX+yjuhXosnAXQoilybqp1teYCVxrMF/n7uu8RQXI6XubbDqqKmXe2LzaMY4Cd+McvZ6Oi5muA24A9lN38ypaQC+q9VXuFu7qgS6EyIoEdCGEyJ+si0wvLdxRBbovVDm4F53DsPT3oggL9DJXOhRVSQGr/zlNAT+Jjy3U+6VfDYzjYpSnVjlGFqpm4Z77GEIIkSMRGZ7JcVsgn9bQaXycu6+xoi/zVYy4MtrdH3mucYuqQM+TXhfQE9JuXnfg3LyuBG7BOXodiV+Xdx6xUhbucQW6YiYhRCYkoAshRP5kTXREGV9XNXxNilhrvcwX+HKdvby4oiVFibW9WoG+FE8B9wDfAS7ECenXAJfjxPUdwP040b3TVKoC3VqrCnQhRGlZv369nZyczBI3ycLdL3xcU/s4ZyESihJryyCgp2l087oceGH8b7+Nc/PagXPz6vTcihbQc69ARzGTECIjEtCFECJ/ZOHeGm8FdPyac2LV3+3PITpPpy/qUucrotqhzImaIhMbeYxhcQmffTghfRyX2HgVcBOwG5cUehiXQOoEuffYS41D3mOpB7oQogJkWR/7toZO4+PcvYwVPbbqrzpFBcN53j9VELiL+n7lNYcp4MfADPBG4Ge4funX4GKopF/68Q6NV0gskxpLPdCFED2DBHQhhMgfWbi3xsukSIyPcxbVQvdwdoqycK+CSG+AE8DXgCFcv/QrgDcB88BDuKTQnlV+lqpZuCsZJIQoO1meYT6KyAk+z93XeYtq0PH7N8PG9LLuXJeF+/LGAOfm9W3gabiY6QXA9cAhXMz0AKtz86pUBXoYhtp0LITIjAR0IYTIn6yLP1m4+0VZA9rV4tt1Fp2nrBXoefdzK1MP9CzjJGPM4KoqfgZswlVXXBkfkzh79x3A0RWOU0T/u0J+bqpAF0JUAFm4t8ZHAd3HmMnHOYvyUFQP9LzjsioJ6DY+9sbHXcAluHjpJuBmnJvXfcAjLN/Nq4o90BUzCSEyIQFdCCHyZzkV6L4lRDDG+Cyg+zRnJYLEavHp+7IaqlKB3myMUeBv42MbrsLiKuA64Enq/dInVjlOpynE9lA90IUQFSBEFu6t8G7uHseKvs1XrIxWra8Uf7emKHE7T5aKMRZxjl0PUXfzuhJ4M3U3r/uAJ5q8txmFuGnFFGHhrk3HQojMSEAXQoj8yZrokIW7f2jOogp0LMDtcr/HIirQyy5uF3V9svysDsbH14GLcWL6y4EbcdbuO4CduCTRasbpBEVVoEtAF0KUnSyOXN6JyCl8nruv8xbVolNrwW7HTFD+CvS8KbICvRWNbl5XcKab1324zcjtxqmEhbtcu4QQy0ECuhBC5E9ip9SOqMviUbew0HXhrBv4nAAT1SGPe7hVJUWrf89z7DKcv2o90JfT/uTR+BgELsUlhm4DbgUexonpuzmzaqJSPdDj36NKBgkhykymmAk/Nx2Dn/GDj7GiKoerR9H3bxHjlT1u6qV4pqgxRoHvxscFuJjpapyb11Hqbl6TTcZhmWOtlNzjmdjCfTHPMYQQ1UECuhBC5E+SMG+3uFUFul/4lhjxbb5i5XTrWZD3uGU/f3qMXksGJcwBv4iPEVy/9CuAdwDT1PulH4pfn7tFYGoc8h5LAroQogLIwr01Ps7d1xjCp+vs01yLJO8K8byoUnV4L49xID7Sbl6vAF4FPI6LmR7CuXkVEsvE5C5uy8JdCLEcJKALIUT+JBXoWQR0H4NHXwV00JyFWAllrUAvqtrZtwr0pRgHvh8f5+JsCrcDLwSewlkV1jowThaKtHAvojehEELkRdt4KO6J7eV60ue549e8fd00IMpFmS3cizh/rwvoCSHwSHysoe7m9QbgtTgR/WT82kpYuMcV6IqZhBCZkIAuhBD5E5HdjtCnxECCrwK6bwkwJYKqS6evbTsL97wou9WhrxbuWTgSH3cDz8Alha4H+nG2708CDwKzHRwzTVEW7uqBLoQoO1meYRFgrLUY49NSGvAvfgB/Y0VRTYqMm8pagZ5Q9o3NZRxjFvh5fCRuXlcCZ8X//ircJuTDHRyzEUPOMZMq0IUQy0ECuhBC5E+moD+uKJCFuz9IUBZVoNPf224/B4oQn/M+vwT0pYlwvdB3A3cA/xyXPLkVeA2uj/oO4DE6m1QpxPYwiiIlg4QQZSfLhuJ07ODbetpnAd03fLvOVaej1zNu29NtyizQyya+PWk3rxuAlwGX49y8juNiph3x6zpJ7i22wjAMrLXzeY4hhKgOEtCFECJ/0hbu7V7XC4FQocQbB8DDuaM5iwrQheqvsiZr8k6iFCVuFzFG7pUHuH5+8zi7wnup90t/GzCDq0jfAezvwFiFXBtVoAshKkDWmAn8FdB93HANfsUQvt3XolxUZVOwYqbsTMR/fgR4Ji5meimuZ/peXMy0k864eeV+bbTpWAixHCSgCyFE/sjCvTW+Cui+bZhQIkhkpVsW7q3GLsP5q1KtkYxTZI+9kzgR/V7gHFxSaDvwfGCUeoXFiVWMA7JwF0KIdmS1cAcnJPvWw9S3+AH8jRV9m6/oLEWspct8/rJXhxc5RnqcCNgVHwPA83Bx02uBW3Abk3fE/77SmKQIC3fFTEKIzEhAF0KI/FlOBbqPFQW+JkUkKIuqsKp72VpbC8NwJIqiTYuLi8/GBeMPAXN5jNeCKvRYh/Inm4ocp5lF4JPAN4FvARfh+v69GGdbeBDX9+9BYHqZ49BkrI4iAV0IUQGWa+HuG7ZHrJuLRDGTqBIrvp+ttVhr14ZhuDkMw7OstU8HxoCnOvbpsqGYKRtVEtCbxUzzuLjoPmA9dTevt+PcvB7AiekHOjBWRwnD0FhrF/McQwhRHSSgCyFE/mRN8vhYUQDxz8fDZBD4eb19nHOVyXQ9oyhaE4bh5iiKNllrN8V/brbWbgI2pM6TPAcvx4mUv8DZwmUea5WUWXyukt1hL4xjgSfi4w7gObik0KvjYxcuYfQo0C4BU4iFexRFEtCFEGUns4BurTVdaCPTbXyNF8GveWvTQPXIdP9aa4MoijaGYXhGzBTHTQOpl0e49ekRXMy0g7qFdpkr0IuKafKmKn3Ws4wzCfwwPs7BbUDeDlzL8t28cp9TnHtUzCSEyIQEdCGEyJ/MFu6eisheVpEYY3yrIPExEeTN9Y2TPRuiKNqUHA3JnjWpl88aY0aNMSeCIDgQBMFofJxYXFw8e2Fh4SzcGvUaXPB9ApcU2pnzNMp+vaqSbErGKSoZlMUicBG3oeNBYC1ug8cVwFtwTgk7cUmhvTT/3LJwF0KIbLR9Thpj0hbuvuGdgG6M8TJWxL/5ekMURWtSMdOpDcZxzDTC6evGMWPMaBAE+40xO5KYyRgzNTMzsx23Dr0EuBm4Cbcm/UXOU6jCvVmV6vBeEdDTPAncjXP0ejouZkrcvA7gYqYHcFXqqx1rRYRhGKgCXQiRFQnoQgiRP7Jwb42vSREfBWXw7zpXhiiKBtLJnrgq4gJgYGpq6t9wuk31eCySH67VajuDIDiREspnlxojDMOzcYmgHwE/wFloX40LuF8Rv+xcnMV7HmJkmau3fapy6OY408BP4mMLLil0Be4+naBeYXGsYRxWMNZyUTWFEKLsyMK9Nd4J6PgZM/k458pgrTXxxuLNqdhpG8Ds7Ox7gMHUy+eMMSdikfxQQ8w0kdowdBpRFA3GG/KfAH4OrMNtPL4aV/k7j6tGX4+rDs5lqjmdtyquWlUZIxlnubG3BfbEx53U3bxeg3PzegwXMzW6eeXeAz2KIsVMQojMSEAXQoj8SRZ/snBvjq9JMF+vty+ULvEVJ3vWpZM9STWEtXYzrvI2Yd4YM/r/s/fe4XFd553/95wzM+gdJNg7KYm9k5IoS1SjauQmWW0tJ+uqJGsn9sZJHEu7m8ROftnfOkrsxEk2jxPJkeTIKrGiShVTjSLFBnaCAEiQIEAAxAzaDICZe8+7f5xzgYshygCYuQBmzud57iMKvLhn7jSet3y/L9R9dvn9/o8YYyEhRIhz3s4YS0ZA6rbQfg0qIXQrgG0A1gI4BKWwSMQKLhG8mOfnxftiqiebptI6rQDe1cdcqMTlRqj3aCP6FRZezkBPacLJYDAYUowNU0AfjkyMHzL59c4EplzMBFzWWOzYrTvjqYoxcO/XwRiLAIAQ4gDn/ALnPCSECA7XWDxKuqCajz8EMA/APQBKAPwe1NihA1CFymTsE72agT5VLegd0q2APp51YlAx0VEAebjczesYVNx0Dt7MQOdSSlNANxgMCWEK6AaDwZB6ElWgJ6K4SDsy2JYPyKB7dr3OhgmEiHy2bZfEFcgdq/ViDNwbdmoV+SUhxGmtjAgJIYKMsQhjDJFI5AEAyMrK2pfMhznIz3qgiuW3AngTKiHkFCrroJJCJ6CC87HiVQIiVaSLWmOqrnNeH69BWWmuAXAL1Hv2oj4npS4zxsLdYDCkAYnGTM53XqaRiQV0AH0zazOJTLvfSYWrsXiA+5bLaj3PdXrMNZ7qlBMzcc6DQoh2xpgdi8Xm9/b2fsnv9+/3+XytKX7456Diox6oGGk9gC8ACAOo1D9LxmNIh7gp1WuYAvrlhAHs0Uc5+t281gNoB+DHQJeGpGMU6AaDYTSYArrBYDCkHqeAnsh5mZoIAjIvSTCVA87xkGmvs6cQEYgozymSE1H8XL0C1+mWTvaEOOc1ccmeNsbYZJsL5rx32gDshiqkXwUVbH8GwB0AjkAlhRon4gGOgFcW7qaAnjipUDjYAE7qIwfAcgBb9N99A6rRoxLKzjCpa5sCusFgSAMSUUdmauwAZGYBPRNjpky8Z8/RjcXFgxXIddw0WGNxqxCiWhfLQ0KIEGMszNik+1g6Ntj79VEB5ea1DsA1UE2fB6CUv6NtQE4HBXq6FLencswEAJcAvIN+Ny+nkL4GwHSomOkoVNE9aZgZ6AaDYTSYArrBYDCkHmPhPjyZmgTLtNfbJIKSBBEJ27aLHKv1QZI9AdfpYVeR/GzcXL2uKe4MYEEVzI9AKdLXQVm7b4RS/B6EsoJL1BrRiwSEmYGeWet0QyUtuwDcDzU3/Uqo5FAX1Hv3MPoV6uPCFNANBkMaYCzchyfT4gcgc1/vTLvfpKMbi3Nt2y512az3uW9hYGOx7YqZal2NxSHdWDwel6tUkug+tgnA6wDegppFvR7K5v12qP3oQQAXErxWOlisp8NoLSB9YiaCck44B2AhgBb9s1sB7ABQAxUzncT4HOcAKAW6KaAbDIZEMQV0g8FgSD3Gwn14MjUpAmTmPRsSQEqZPUyypxD97x0JoE0nec4xxipdBfIQ5zzqwcP1ugg/2HohqO71XwNYDJUU2gEVdB+HSgqdHeJ3vcKrz/tUTmY5eJkM8mJmuPPafwSVuJwNVURfA+BqAM1QSaEjADrGuogpoBsMhjQgYQt3ZK5zV6bGD5l634ZhICIhpSyybfuymGmQxuKIy2r9rCteCnrYWDzRzcsWlOr8GNSs9rVQTcgboPajB6D2pN0T9QCRPuQ94MgAACAASURBVDGTF2ukW8zkrHUJKmbKAbACKm76LIAolJvXYYzDzUsr0E3MZDAYEsIU0A0GgyH1SCSwsdMBW6YmgoDMS4pkZAIsA+cXDgoRcSll4TBz9bJdp/e4kj31ccmejglWkSc7aB/u/ZHIe0cCOK2PPKgC5XoAq6CK7AcBHALQOcT1p7LSIZ0U6KmyCYzHy6QT0J94uqCPNwAsgUoK3QDgZqhGj0qo5FDvaBYxBXSDwZAGjKgIc+17MnFPmYnxQya+3hNdZJ1USCmz3TFT3HiqIgzeWHyeMXbY5b7Vxjkf1b4qyST7/TvS9RLZ47ZBNR/vgmpAXgfVfHwLlNL3IIDaQa6TagV6Oli4e6XSTxcFevxaTszUDWCfPkrQPy99DVQ877h5NY1mATMD3WAwjAZTQDcYDIbUk6gCPRMTIoDeiGdgYTXTEiOZdr8gImFZVoW7QO4kfKA6/p2GGQLQrovkjUKI43FW64lakKcLI71XEn0vhaEUvx8BmAeVFLoOwHYA1VBJoVPwrpsemNqJIK/WcEinZJD7s+5GAqjSRxaAq6ASQvcAuBPq/VkJZVs47PtUSumsY5JBBoNhKpNozJSJsQMYY5SB951xMYQmo15n93gqV4G8dJjG4hDnvCEuZupgjHm5rx81k3BWOqA+Y9X6yIXai64D8DCAdvQ3ILdPwONKFekwx91ZJ51iJmDoRuoQVLPHLig3L+d9eg2Ue0IlVEF9sEb5Aei4ycRMBoMhIUwB3WAwGFJPol3zGWnhnuEqEnPPUxgiYlLK/MGSPQBybdu+3rbt6/XpUVey5yTnPMQYCwkhQpzzdsaYCeD6Geo9Mp73jjNT7XUAK6GC7fugiuyVUImhqa4QT5cCulcJJ2ctLxXow63VC5WcPAQ1pmEVVGLoQQARAEeh3qsNg/2yVlIAJhlkMBimNonEQ8bCPTPJ1PtOC6SUAZeK3LFbLyWimQCyu7u7v6VPJQAd7sZiZx65ECKYgY3FY2Wsn5cIgN36mAPl5HUtlFNSDZTFu2PvnmoFeiqZ6nGfe510ipkSXcvt5rUYKma6Eco94Qz63bwGHWdn27ZRoBsMhoQxBXSDwWBIPQlZuOtzRIofy2QkUwvomaggmXIQkc+27ZLB1BBEVIyBe6lOnexpIaJpnPPTfr//YyFEkDEWmaSKg2QwlZRBvQD262M6VFLI6VzvhlJW+AHEUrB2OiRqvFwj3azigcTdDjoAfKiPGVBWhasAbAbQCmVVeBjKehMAEIvFRruGwWAwTEYSSWhnauwAZGABPUObrafS3hrAgMbiocZT5blOjzHGgoyxEIAWIpodCAR+wTkPCiHSvbHYy9d2vGvV6+N1qDnU6wHci/4RQ6VQ+9JkYyzcR7dOOsVMwOjmrdsY6Oa1HCpu+jSAu6BGEVRCjSLou6YuoJuYyWAwJIQpoBsMBkPqSdjCPUMLqpmYFAGmYGJknEzK+yUiEFGeUyQnovi5egWu0y2XirzGUUPoZE8bY6xvbmdXV9dSznmT3++v9/6uPCUVn9tkWbiPRDNUQmgngCsB3AFVrPwOlNr3AFRnezJJ9ecgHZI06ahAH8rCPREu6mMngEVQSaFtUKMIzgE43NXVdVJK6SQz0znpbDAY0p9EFOiZGjsAGVhAxySNITxg0r3OurG4OH4OuStuGqyxuFUIUc0YC2oVeYgxFnYai3t6eq6xLGtGIBConpCb8o5UvZ5efD6iUG5dBwFMA3A9VEH9QagC+0Go2GlQte84mMpxU7oV0CeDhftI9KL/fVqEfjevlVCuc0fD4fDRnJyceqNANxgMo8EU0A0GgyH1OJ2NxsJ9cDI5CWbu2QOISLjn6g2S7Am4Tg+7iuRn4+bqdblUMIbUMNz7I1XvHRvAMagAWwA4D6VKXw9VZD8IpfaNjHMdY+Ge+BrwYB1nLS/UB8m4J4KyzqwB8ApU08fqSCRy59VXX33X6tWru6644gqcOnVKALCGu5DBYDBMYmwYC/fhyMQCukMm3feExBu6sTjXtu1Sl826exa5u7HYdsVMta7G4pBuLE6Fm5Ohn5E+D6nas7cA2AdVQH8VwDIAdwO4DaqIfhAqlhoP6eCqlY4FdK8U28m4p3YAH+hjJrSb1xNPPLHl1Vdfta+44oqu/Pz898a5hsFgyBBMAd1gMBhSj6NAT+S8TE0EAZmVFAEyOwGWdKSUOS4VeXyypxADLZTbdJLnHGOs0lUgD3HOk905n9YQESbAmj6V8/aiAH4NYBeU2nc91Cy1m6Es4A5CWcCN5TF4YRWYDkkaLwvoXq2TbHv1KLSNe3d3d8mDDz54686dO5ecOnUKABoZY78A8BSA3URkmn4MBsNUYsTvrAy19HbIxPghk1/vpENEQkpZZNu2ew75UI3FEcdqXTcW97lvmcbijMb5LFYD+AQq1l6L/tFYl6CcvA5DKX/Hev2pHDelYwF9Mlq4J0KjPnauWrVqS3V19eZ33323uKen5w8YY1sB/BzAc0TUNvxlDAZDpmIK6AaDwZB6ErZwT+CcdCRTkyKZlnAY1/0SEZdSFg4zVy/bdXqPK9lTH5fs6TDJnqTjZTDtFW61by6UOn0dgIehOtoPAjik/zxZ8EqtkS4FdK+V7imhrKws9Ed/9Ee/ePjhh7O3bt36HSL6BwAPAfg6lOriulStbTAYDCkg0aZjZPDoq0y774zbt+vm1DG/zlLKbClliW3bpa7GYidmKsLgjcXnGWOH49y3TGNx8knHuKkDwHsA3gewECpmugmqAfkUVDG9BpPrs2wK6JNznVSuJe++++7dd9999+477rjjrkOHDp0AMB3ATwH8LWPsNiL6dQrWNRgMUxxTQDcYDIbUYyzch8GlXs24e4e55wFIKQNOcTxOSV4CoBgD5wi3a9vARiHE8bhkT48XN2IAkF7v4aGC9QiA3fqYA5UUugbADVDJoANQyaGR5qh5kXhIhyRNOhbQxzrLL2FisRgHYBPRHzHGvgfgU1BzKscMY+xGAF+Eer/PBhCCsu38X0R0YJwP2WAwGAYjEdVZRlu4Z2jjAJBee86RGHbPQERMNxaXOs3FukBeOkRjsWO1fiHOeauDMeaVLXNGM0Gf21S6dg12fYJy6qoFkANlm70eqrGzA6r5+CCAkZS+Xu3RTQF98q0DeBA3aUHFHiL6MWNsFoD7oWKc8VzTxE0GQ5piCugGg8GQekajQM/ERBCQmc0DmaYgIQCQUubGYrH5QyR7cl3nR13JnpOc8xBjLCSECHHO20yyJ6OZSPVCvT7egJr9tw7AvQC6AVRCJYWah/n9qWxF6JAuhW13Q06qSfncQMuymPO9SEQSahTBePk6VBH+RwCOASgH8B0AHzPGbjUqDYPBkAISVqAjs/bRDpkaM2Wc4wARccuyKlwxU+kwjcUd7sZiZx65ECJoGosNE0Q3gD36mAVVSN8K1eB5BqoB+SQAa5hreFHgTjXpUkBPeVHbRcrvSUoJ6OZ3ImoA8H+ScFkTNxkMaYopoBsMBkPqGc0M9IxKDLjIxHufTBZmSYOIfM4scvcccillOQDEYrH7Xad36mRPixCiyp3sYYxFJmC2tmFyMNRnI9VviNEE61GoYvlBqOB4PZTN+1YAF6CSQkf1ee7rpxIzA31yruOs5YUCPdlF+t8mohb3Dxhjb0DNvPxjJKdIbzAYDG5MAX14TMyUJmgVef4Q46lmAhA9PT3L9Okx13iqU/q/QR03tTPGRnJBMqQvw8VNXivQh6JBH28AWA4VN30OQA/UnPQDAJoGuX6qMQr0ybeOs1ZKG4+llBwju8eNFhM3GQxpiimgGwwGQ+pJyMKdMSYzrbPeRcbZEY53tt1EQUQgojynSO5WQ2hFRIHrdMulIj8npSzx+Xw7fT5flRCijTE2XMd5OjDlXt8xksxgmmHqPW+XALwJ4G0Ay6CSQncBuA2q+/wAgPPwbtZeKknHAroXbhYpV23Yts2SnUCPTwLpn4UZY8cBzE3mWgaDwaCRGPn7MqMt3DH19knJYsrdt24sLnaPp3K5bxVjYE7WaSxuJaIcxlg0EAi8KoQIMcbCprE4bUjWfnAqviFiUI5dlQDKoJy81gLYDFVgPwjgCLxpOgZMAX2063jlAOhJ3IQkF9BN3GQwpC+mgG4wGAypx1i4j0wmJoMm7T0TkbBtu2iIZE8JAL/r9LCrSH4mbq5el54vBcuyynp6etYIIep9Pt+libkzT0lLtcwgePkeTnUiYrwJCBvACX0UQiWEnMTQJah5lKnee6dDksYo0MeAZVmeJLYYY4VQTSJvpXotg8GQkYy4P3b2liOdl6ZM2vghVUzm13uQxuIBSnIMbCy2XTFTjeO8pVXkIXdjcSQSeQAA/H5/vdf3NAGYmGnqrZeMPXQr1F7yHQBLofaWdwDYAeDiOK+dKFO9sdlZJy1iGS/XklKauMlgMCSMKaAbDAZD6klESQFkYELERSY2D0xoskBKmTNMsqcQA9WZbTrJU8cYOxRXJI8OvcqgZOp73JA46ZBI6wDwHoD3ASyACpxXAtgElUw9CGXnlmz1vldFZy/WSLcCeqpnoHNnBnqK+TsAuQD+3IO1DAZD5pFI3JSRM7GBvmJypt33hO4LiUhIKYts274sZtJxU8B1esRltX7WFS8F3Y3FiSyLzHudDclnKsRUEsApfRRAjcS6Gur9/ztQMVMlgK4krplOCnSvZpN7WUAHUm/hnnQF+hCYuMlgSANMAd1gMBhST6IKdJnAOelKJiYJUnrPRMSllIVDzNUrgVLDOvS4kj31rkRPiHPeMYpkj8HgBZNVgT4YBOCMPuZCKdFLATwIVWQ/BJUYakvCWmYG+uhwmra8+H5LeXIrFoulPBHEGPtTAA8A+B0iOpTKtQwGQ8aSyHelsXDPLFKuQJdSZkspS2zbLtVFcnfMVITBG4vPM8YOu2KmsTQWGwxTkVTt1TsBfACgF2oUVgOAGwDcBKAKaixWNcZfXE23huC0iGVc6yDVa6VoBvoATNxkMKQPpoBuMBgMqSfRoD8TVdgOmZgMAsZ5z1LKgFMcdynJS4ioFCrZ4w5A2rVtYKMQ4nhcsqdnvDeSAKYIn754aeeWalJ5LwSVCHoHwCwoVfoWAJ+CKrAfAHASgDXUBUbAFNAn5zrOWqm2cE+pAp0x9jiAPwbwx0T096lax2AwZDyJunYBmRk7ZGrMBIzjvomISSmLXI3F8eOp4huLHav1C3HOWx0eub0Amfs6pzupaNYdDC/27KmOm16A+myughqL9QCUEt1pQA6O8drppEBPNwt3T563VMxAd2PiJoMhvTAFdIPBYEg9xsJ9ZDLx3kd8T+hkT8EwyZ5c1+lRV7LnBOc8xBgLCSFCnPM2D5M9g8JY38ubaa9zupOK13Oi3iOpXtd9/QZ9vAFgOVRS6HMAegAchiqmN43j+qkiHQvoXnw3emHhnrJEkE4CPQ7gMSL6y1SsYTAYDJpEvitNAT2zSGg/4Gos7msodmInAMUY2Fjcod23GoUQx5155EKIoEeNxSNhGo/Tj3T63HoZM/UA+EQfM6AakDcB2AagDipmOo7RNSCbAvrkXgdIcdykR8CYuMlgMCSEKaAbDAZD6nEs3EdSl8tMnOWnydRkECMinzOL3En2xFmtC9fvdOpkT4sQokone4JCiBBjLOIqUk9GTCLIMCj6e6+QiIoBLGCM9RCRwNBBbarVDqlisMRDDGquXyWAMqhC+loAmwE0QiWFjkBZGSZCOiRp0lGBnnLbQ61AT3oiiDH2fagk0P8ioj9L9vUNBoMhjhG/K11NoZno3JWpMROIiNm2XRDvvuWKmfJcvxNzjac65cRMukjenop/L1NApr3OhgQgoiz9nq9gjE0jonyosVAT8nBSdN2h4o2LAF4F8CaAq6CK6Z8BcAdUvHQAKn5KFFNAH906XjQdT2kLdxM3GQzpiSmgGwwGQ+pxNprGwn1o0jYZREQgojxdJO+bqyelnA8gJxwOf891uuVSkdfEJXvaGGNjtXY2GCYUXRAv1kXyQiIqIKI8ALlElI3+z3+2EKKXiD5PRBeklCcAtOq/81LtkCqGSwa0AngLyuJ9KVRS6A4AO6CUFQeglBZDYSzcJ+c6zlpeWBEmNbHFGPs2gP8J4DUArzHGtrj/noj2JHM9g8FggFGgj0TaxkwAMEhjcYmUsgQALMu627Isd6zcqeOmViHEaZeKPMQYC0/yxmKDYVCICADydFNIEREVACggohwAOUTk1+dxzrmfMXYjEYWI6DQRueeDp3LvOdExkwVVMD8CoAT9DcgboYrsB6EcvYZykzAz0Cf3Okj1WlJKIMkFdBM3GQzpiymgGwwGQ+pxFOiJnJepkf6UvnciErZtFw2ihnCs1v2u08M62dNNRAV+v/9l11y9LsZYuiu1p+zrbBgSRxnkB1ASVyTPhUr2ZEG/9kQkAXTroxlAJ4B2AG0A2hljRQBWMMZmM8YWEFE7EVUTUYt7vVTeS4pI9L0vAZzSRwGANVDF9NVQs/4OQs3+6xrk+l58f6SLtbrXBfRUW7inQklxF9Tzc5s+4hGD/MxgMBjGg5mBPjxTPWZyNxY7BfI+9y2ofY+DzRgLAQgBAOf8uM/nO+IqkqdzYzHpQqoh/aA4961CIirUSvIcXSgXrpN7oQrBXVDF4Q6ouCnEGLMYY0uJaAmAzUS0VjcgH/fqXlJ03dF8x4Wgmo9/DWAxVMy0A8CtUA3IBwGcxcDH6mUM4EXc5NU4qrSxcE+RAt3ETQZDmmIK6AaDwZBiCgoKqLOzM5Fkh0zgnHRl0ieDpJQ5wyR7CjFws9/GGAtyzusYY4dcBfIQ5zwKAD09PVdblnV9VlZW5UTdk8eYLNAURyfycuIUEaVEVBGNRh9iLqkPEVm6SaQbKrHhLpIPazOoG0w+ICLBGFsqpVzGGLtGSunXj6EwRbfohcJ9tJ+DTgAf6GM+VFLoegA3AqiCSgqdhnf/fqSTAt0Ti0DXWlNOgU5E25N5PYPBYEiARL4rJaAUmCl+LJORSR8zuRqLB8RLrrgp4Do94rJaP6tjpWB8Y3FXV9fjQogzgUCgamLuakKY1K9zkkm7e3W5bxVBFckXAEAsFruXc97DGOP9p1IPVJG8DcB5qJipHSqGGrJRRIdeJxhjJ4iojIhWMMYWcc5XSCmL9DWGG4s1ViZjzCShYqLTUOMcnAbkVVDPo9OA3AkzA30spDyWca2DVK+lm1hM3GQwGBLCFNANBoPBG0YsbugEQSYmgoBJkAwiIi6lLBxmrl626/QeV7KnPi7Z0zEKFXnaJQtGQgcrhkmKLlAXHD16dM4LL7ywrKOjo2DlypX2pz/96Z6CgoIsIvK5zo1yzttt254djUaXMMYucs4Pc84PM8bCiSpnHKtCnWQtJaJSqMJ8CYBSADmu0y3O+ToA87Uq/TSGSSyNgcnc6FGnj9cArIRKCt0PpUg5BCAL6ZGkMRbuY8C2bTZFZroaDAbDcIyY0HbtszNxTznhMRMASCmzpZQltm33jady3LcweGNxiHN+njFWOVhjcYJM+H0bDG6IyB+JRMqee+65Kw4fPjyzqKjIf88990RWrlzJBnHfijHGwpZlzQXQzTk/xTnfzzlvwCgKeUTkczndOYcTMxXDtb9ljPVyzu8lonop5TFoN4ckMlnjpjCAj/QxDypmug7AdgDVUNbvQPoU0NNRgT4lZ6AbDIb0xBTQDQaDwRsSUQdOioTIBEFeFFallIH4Arn+bymAIgzseG3XSthGIcSxuGTPUPO0RkMmv96TEiklnn/++dn/8R//saynp8e/YcOG+kcffbSqqKhowiwio9EoO3DgQNHFixdzSkpKohs2bGjLz88fV7CnP2vx88j7bANffvnl/P/92GPlhV1dfDZA//eXv8Sz//IvXX/zj/+4b+HChS3oV0REhRDgnPtt214ppdxg2/attm1fzTk/oI8OZ01tT1jqSrA6xfJ4NVIngCBjrJkxdgpAkHMeZIwFoYrli4loKYANRLTGZVXYivExGdUUg9EDYJ8+ZkAlhTZCNflYUFbvJwDEkrBWPOlYQPcq6TQVLdwNBoPBa4yF+/B4Ej8QEZNSOiryvnhpmMbikC6SH4mLmToYY8n49y/T4qbJWpjso7a2NuenP/3p8urq6vKysrLwI488cnzbtm3BiXxM58+fzz5y5EiRlBLLly/vWLRoUfd4rhfnvlVIRIVQ88hzAWQHg8Gc3/3a16afP3QoexkRqojolX/7N/vBRx89/eijjx6Gy32LMQa/3w8p5UzbtjdKKVdJKdcwxpxCeo3THEREWYMUyZ2Yye3EFYOKmYJQSvSg6+iAirWuco3FChFRjW5AHs+ecarETABwTh/uBuTP6r9bDTVObLwx5GB49X1lZqCPASklg4mbDAZDgpgCusFgMHhDIokDY+E+3ouoZE/BMMmeXNfpUZ3sCXLOT3DOQ4yxkBAixDlvS1KyZ9iHm+LrTzZGfb8NDQ1Ze/bsKY1EIr45c+ZErr766mAgEEjZ8/b1r33t2g9eeWXrYssSpQCeef/9ta+8+OKFX7z44i9nz57dm6p1h6K5uTnw1E9/usw+dWpaiWWJGs7lR/PmtX3+y18+uXTp0shwv6uV4iUu28ACIsoDkE1E2ehvFpEu28BgS0tL5Iff+961d7e3x/5Hfn5nNkB1ts2/fOpU/h98+9tFz73wwofxazHGYj6f7yARVdq2vVRKuUFKuU1KeT2UOlpCWek5c78IKqEUZIydZ4xVOo4O+hip8FvFGKvSn+nljLE5jLH5elZ6DRFVYeyq9FR/LpN9/YsAXgXwJoB7ASwC8BkAd0CpKw4AaEziel7ZxAPpU6h31ppyFu4Gg8EwASQaMwGZ6dyVtEKyq7G4L2ZyGoyhlKzuxuIOvVdrFEIc1zFTMImNxSORaXETMIrX2bIs9vHHH5fU1dXl5eTk2Fu2bGlNZeyya9eusm9+9auf94dCRWsBdhqgr7z++savffvbr37rW9+aEJv9F198ceaJ115bVNjens2J2JHCwp4FN9549gv331/P+dBfFY77lo6ZilxF8mwAufHuW1AxUwRAyw9+8INZrXv2zPh5Tk7bSiFiMSI80dWV+9Mf/Wj2+vXr39m6detlim/OeSNj7GUiet+27U1EtMK27Yds244C6Abgx8CcRQ/6i+R1rpgpyBjrck3QGoxW11isZXpW+kZXA/J4Vemp/Fwm+9q9APbrYx6A34SKm34Hyjb/AIBjSF4DcrpZuHtVcPakwVmPgDEFdIPBkBCmgG4wGAzekKgCPRMTQcAokkFE5HNmkccXyHXCR7hO79TBZYsQokorI4JCiBBjLDJCwOkFE/4AJoCE7nnXrl1lbz311FUlLS35BYzhfZ9P7n777Ytf+eY3T5SVlSVdWfvSSy/NfP+VV7Z+H8B/zcuLcMZw3LLEg3V1sx977LHN//zP//x+stcciV8+88z80iNHZn5m3rxQRW5urKO3V7x05kzpS08+ufT3Hn/8sBAiAFUkL9RJnz5FhE76AOizDYxAJWCaoWaQu+eR9wXd//qv/3oVa2vz/UFeXme2/vl8IeRv+f3R7x85srixsTG3oqIibwhFhNs20IYqnvuhrMW7GWNVnPN9OnE07oBVJ48+JCLOGFuiVenriWg1ETVIKU8AaBnNJcf7mCbw+haASwDKAfwcwDoAa6GU6U1QSaHDUO+B8ZCOCvS0mBtoFOgGgyFNMAr0YdAK1URjJialzB9iPFUpBhbpYq7xVKecmIlzHhJCtE+GESEZNgYq4T1De3u77x+eeOLK6OHDMyssSzQRsUOvvNJ1/YMPnrjpppsupeLB/c/vfe+mpaFQ8b/m5kamcU5RIvxhJJL7T088cet9991XN2vWLE8bjw8dOlR46le/WrKDc9qyZMklxhgONjfnv/raa4t3z58fvuaaa9qgiuPF+r9u961sqPwBAWBE1AtVyO4C0ID+uCkEoG/kgJQS77322te+zlhspRAxAPAzhv+Wlxd5vqur4Pnnn1+yefPmk3HxkttuPct1CxG9vqMur2eMVWoXvHGp6AFAf377ZqUDcFTpTgOyMxYr0c/5VI6ZgH7F+UtQe/T1AO4BcDuAo1Bx04VxrpGOBfS0mYFuFOgGg2E0mAK6wWAweIOxcB+evnsnIhBRni6SDzZXL9/1e5bLMrAmLtnTxhibMOvtBMjk13tYWltb/W8//fQVW0Oh3LsWL74UEIIaw2H/vx06NPuF55/v/MpXv3om2Wv+6le/WjrPssRv6eI5ACz3+ex7o1F68oMPlgPwtIDe3NwcaDt6dPqtM2ZEp5WX58tAICfP58vZUV6e+8vW1kXnzp1bOXfu3L73DxHFGGM9RNQNlRRwF8m7El03Eon4AwArDAR8JISA8mj3Tff5fPlq5t5/t6y+j5XbNvC4Sw0RZIx1MMaIiCClnC2l3EBEq2zbXiWlPKUL6Wdcc0zHjHaLcKvSr2KMzWWMzRuDKj3Vs/C8ULiHALwD4NcAFkMlhXYAuBXK2v0AgLNjfCymgD72tVKqpDAz0A0GQ5pgCujDMyB+GKSxuC9m0sVCd86vU+/VWoUQp3XcFNKNxeFJ0Fg8HJkYNyV0vy+88MKcwL59c39r9uy22fn50ZiUeP3cudJdzzxz5YoVK/bOmDFjNHPmR+To0aP5jWfOzPkTvz86jXMCgABj+MOcnO6XwuHcZ599dsHv//7vn0rmmiNx8MCB0nnhcM7Vq1eHKRCYTn5/ztqKipwL1dVFLY2N8y3LstG/hyW3+xZUzNQBFTO1IcGimpSSWdGov9znAwKBLFIxk/AL4SuzLP+sWbNusyzrDtevOO5bDYyxY66YKcQYi0I9sGzbttdIKTcS0Z2WZV3NOd8nhKhkjA3rPpYojDG3Kn2pVqW7x2KdQOKW5qnaQ3tVrLWhYqNjUI3ga6GakNdDNZ07DchjaWIwBfSxr4NUr2UUED5EbQAAIABJREFU6AaDYTSYArrBYDB4g7Fwd0FEwrbtYifZAyDXtu2rwuHwUl0A87tOD7uK5Gfi5up1JaMAN0FM1cc9VhK+33379pXkNTUV3L5wYWtACAKAmXl5sa2Fhd0v79s3s+eLX6zLzs5OajEqGo2KAgAiLnlYxBhZlpWy/ZJW0zgK8iKtJs+3LCuvYsaMmSUlJWTn56t5eFLG8oGYjETs9vb2C3PmzLmIfkVEj7YgTGRNAMjVDSp9s8iJqOTTn/502dvPPZfzCpB7d36+kq/btvViVxcrXro0VF5e/p4Qwkn2jGQbCMYYhBAXhBAXiOhN27ZXSyk32rb9X2zbDnHO9wshDiYxKRRijH00RlW6F9+/XhboJYDT+sgDsAYqIfRFqAThQQCHoJKGY10jFXiVcPJE4aDxwsLdJIIMBkM6MOL+Tu89MsK5K66xuMS27aUAWDgc/k0dMxW4Trdd46mcxmJ3kXwyNxaPRKbFTQlhWRY7s3fvzNvy83tn5+dHAcDPOXbMnRuqrKkp/+STT0rvvvvui8lcMxKJ+BgRKxRiwGe1gDFiAOvp6Ull3JQFoJiIitzuW8WFheWl8+cXWtOnx9RpBNh2NF8I2dza2kVENeiPmQa4byWwpo+Iil1Kcqepv3Tr9u0lr7z9NvtcXh6yOAektPd2ddl1gH3f4sUHhRDVrhFVI+7RGGM9Pp9vDxHtkVLOk1JulFLeJKW8iTF2XDcgn0tGs4t+PCcZYyfjVOkLiKhtBFV6OsRM8Wu0QTUf74JqQF4H1Xx8C4CTUHFT7SgeV7oV0FPupqXxxMJdSmniJoPBkDCmgG4wGAzekHEW7lLKHCfZo1Xk7lnkhRi4OSYAnHNexxg7pJM9Qc55G+c8qV3zk4yMaJiIY8R77u7uFgEinuPzDQjS8v1+G9GoLxaLsezs7KF+fUxs3br1/E/femvjnljMt8XvtwCgiwjP27ZYtGJF7XiurTuci+OK5M488hy4AlKtiOgtKioK9WRl0cnTp9n0WbOaWW9vBFLalc3N+U0AlZaW7oea5TbUmkzbEzoqpL5COZRtYMB1eheAYHV1dcd/PvdcLzif9gcXL+Z92NRkL8nK6nlfSvFxVlb0G/fe+5+BQGDM6n+dFNpLRHullHN1Umi7lHI7Y+yETgrVJSkpNJwqvUOr0k/hclW6F8maVF5/qMcfBvCRPuZBJYW2AbgBQDVUUugURk5WpFMBPd0s3I0VocFgSAcS/a6kdLH01o3FRY7Vuns0ld7DuPdsUQBgjLVzzs+6Yqap3licCGnxeidIQq+jZVmMent9+X7/gH//A0JQDoCenh4xxK+OmbVr17bnlZWFnm5qKt/m81lO8/HPe3uzYoFA9Lbbbjs/1mvrJt88HTMVk5pHnq9HVOUQkR/9VusWY6ybiLqLSksbTgeDPVc3NDTnEIV5LNbTa1nsSF3dtPz1608AqB9h3YA7ZqKBI6qKXKdaAILRaDT0wi9/2RJubOzd391dfu/p0+w2vz/aLoT9PBEvWb68+o477nhd6Ebw0aIbkM8JIc4R0eu2ba+VUm6wbXuVbdstugG5kjE23tFMznqDqdLds9KHUqWnUoGeSoa7PkHFRtVQYy7WQMVND0M1YTgNyO0JrpEuBfS0snDX+wcTNxkMhoQwBXSDwWDwhkRs56aUAp2IuJSycIi5eiUA3BXObpeKvD4u2dMRiUS+wTmvycnJeWOi7sdrRjPDME1IOAhasmRJZ2VeXu+xYDBnRWlpNwBIIhwJBvPy1q27UFBQkPRg58tf/nLtS889d+Y3T55c9OloNFDKmHxZSnG+oKDrJ9/5zp6Rfl8ndIq1XWYhgICUcnUsFquASnwydRoRlGVgN9TM6k70W623Qxcw/X4/5q1ZM/29U6eu6jh2zL+wsDDrYiQS2B+Libm33VY9c+bMXiLiWoHRp4hAf5G8BAP3eY5t4AXG2BGX3XqIMRatqqrK+/H3v/+pNc3Nvn+cP7/quUCg7NWLFyvaLMs/e9my2m9/9at77r///jEnxNzopNB5IcR5nRRao1XpX7Jt+5IrKTTumX96vXhV+hIA64hoVZwqfSIL3F5e/5w+XgewAkqVfh9Ukb0SKjE01MzMdCygp1Th4FrLzPIzGAyGkUl0fzyl4iYpZbZt26WuxmL3eKr4xuI2HTOdZ4xVut23YrHY8lgsdk9OTs6LaV4sjyfT4iYggfvNzs6WhYsXB4/u3Ttvw7RpXc4Yqqq2tuxgdnbsmsWLO5P9oAKBAH3p619/78c//OHdnw+Hc7cLIU/YNn+dc7p6x45P1q9fP6yzkS6SF+qYqQjAAgD+aDT6BcYYx8B55FGomCkM4CIGziPvc99avny5f3dp6cqfHzlSvq6oyCc4zz8YCuVemjcvePt117XodXNc7lsDGoyhnJocetE/oup83IiqTsYY/vdf/MXqzrfeWv7lvLzm+6+66tI/1dXN+v+6uvJ9ubmRG+68c9+f/MmfHPDFNYKPFcZYxOfzfUREu6WUC3QD8q1SypsZY0c55/s55/UeqNLbiei0VqWnQ8yEBNaIANitjzlQMdO1UA3INVAW76cwvErfFNBHvw5SvZaxcDcYDKPBFNANBoPBGxKx7Jp0iQEpZcApjsclfEqhAl53h2i7DjAbhRDH4qzWR+qOnnT37gGZlPQaFStXruz69aZN9c/v2rWwrrMzuzQrK3aioyP3ZElJ+K477khKETee7Oxs+dxLL730+OOPb3h5166roj09WYtWrDjz97//+/u2bdsWBFTixUn2aEVEgUsREYBLEcE575RSlvX29hZphdAhzvk5qIJ5Qq/9jh07mgOBADu+Z8/i4+3t03IWLgys27IFGzZs+FQ0Gi2BmtXmfAYlAMci8AxjbL8r2dM2km3gq6++Omdmc3Px786f3xTgHBuKi+v/7Ior6h+vq6vw3377sWQVz+NhjHX7fL6PiehjV1LoljirwvOpVqVLKdv1a5nq76HJpHDvhUr8HAAwHSoptA7ANQDO658fh1a7uUi3Anq6zEDnqV7DYDAYPMBxphqJSeXcRURMSumoyPvipSEai3tcVusX4mKm9hEK4+7575kUS2TSvY6Km26//fxLNTWlP6uqmr68qKi7LRr17e/t9RVcc83ZtWvXjmZMT8L89m//dnVJSckvn/zZzzb8pK5uRn5JSccDv/Ebld/97nePA33FqSJX3FSg3bdyiSgbA/esHIAVjUavZIy1cs5PcM6P6gJ2LJHHU1paGnvgG984/s7OnUt2VlXN8/v9uTO3b6eHr722tKysbH00Gi3FwM9gGCpuCjLGaqAL5pzzEIDIcHHHmTNncur37FnwlcLCyHWlpV0A8LmZM9vebGkp/HkgYH33u989WFpamtDjHg2MMRJCnBFCnJFS5kspHVX6Wtu2m/Ss9COMsSEdyka5XrwqfTG0Kl1KKXXzQio/l5OhgO6mXh/uBuR7oRo8KqHipsHGhKVTAd2rpmOkei3deGziJoPBkBCmgG4wGAzeMCkt3HWyp2CYZE+u6/SoK9lzgnMeYoyFhBAhznmbLk6N+aEg8wrogLnnIfnyN75R/avZs8Mf7t49M9bVlV2yfHndp3fsqB9J1TAeiouLrSeeeOI4EV1Af5F8o2VZ2VAJHx/6i+QxqIA5AhUsd0CpyNsARHw+H6SUxbZtb5RSrpdSLgFQI4T4hHNe5U6OatvAvpl6bkXEtddeW3jttdciFovB5/NZTlEcSh3gVkR0jOcz2HLuXNEaIewA7/8K8nGOK/1+a8+5c8VjvW6iaFX6WSHEWVKzPp2k0Grbtpt1UuiwkxRqbGzMeueddyqaamqKA/n50VUbNrRcd911rZwP/RU6iGK/QCf4pgFYpB4GW8AYu15KeRxDz0of820m+XqDMdYESjNUQmgngCuhkkL3ALgdwFGopNAFeJOk8Wo2eVpZuNu2bRToBoMhHUg0JvA8dtCNxU7MVOpWkmNgUyMB6ND7s0YhxHEdMwUTbCweDncBPdPIpHtOeM+wfv36DvHNbx5657XX5rxYU1PqKysLL9uypfEzn/lMw3D74vHywAMPXHzggQc+dNy3iGgaEd1hWVYOqVnlAPoU5936aEGc+xbnXPr9fmHb9pVSys22bW+zbXuNVlbv55x3ua7F9F7+shFV06dPL7n/oYf8tm2DiODz+TqgCuNNjLETrpgpNJ4ic21tbX4gEsleM2NG0P3z1QUFEX8wWHjmzJm8ioqKlI6g45x3cc4/IKIPpZSLpZQbpJR3SClvZYwdEULs45w3Asrm/7333is7dvDgtFg47J+5dGnoxhtvbBrpMRJRtv6eK4N6jgO62bgcgB8AGGO3AzhNRFVI7h7Uq8/6WPbmUSi3roNQMeQ6KJv3rVAF9oNQsZNRoI8NryzcjQLdYDAkjCmgGwwGgzckUkCXgAoyk6G2dCAinzOLfIi5eu7ZaJ06sGwRQlRp+8CgECLEGBu2G3u8DxOZlRQBMk9JMar7zc7Olvfdd9+F0C23ND311FMLjx8/Pu38U0/ltbS01O3YsaN5zA9Czbsq1O99RxGRD6Uiz0H/+5ARUS9UsqcLQAMG2gaOmBjhnLdxzt+SUv5aqwQ227Z9v23bPfoaEirhOpxtYJAxFvT5fCHHNjAV5JeVReptW0giOPaPkgj1sZgvr6wsnJJFh4AxFvb5fB8S0UdSykU6KXS7lPIWxtjRhoaGIz//67+eVn7uXNm6rCyrLRbju/fvn3P2zJnqLz7yyDkiKo5vRHDZ2scr9oMATuj/9jLGChhjsxhjc4iok4aelT6mW8PksHAfDhvAMX0UQSWF1kEV1Jv19buG/O3kkK4K9FQX0DmS8z41GAyGiSRRBXrSLdx1Y3H+EOOpSjGwsTjmFOM456c4586fQ0KI9pGcf8bzMPV/MzFuyrR7Tvh+16xZ07lmzZoTO3funPb666/P37Vr17ympqbchx9++Ex5efmY1dBElO2KmYZz37L1PPIIVGHciZnakID7FmPM9vl8xwAcs217ppTyWinldVLK6wEEoeKxHAzcy5O+fpAxVscYO8gYC/l8PqdInpI90bRp03qsrKzYme7urDV+f9+4qbPd3VmxQCBWUVGRlLnkiaBV6dVCiGotilgvpVxvWdZ6AA2c8/1P/su/dLa8/faCKy2L5/t8surw4dn/d//+GV/85jePzJkzhzu29nExUynU8+0QgXqeWxhjp6BiqCzG2HQAG4hotR6LdRyDz0of9a1hcrl2DUULgDcBvA1gGVS8dDeA2wCc1eeYAvro10Gq1zIFdIPBMBpMAd1gMBi8IVEFOjDKzSkRQSs2L1ND6AAo33W65ZpFXuNYlelkT1uqAs1EbgOZlxRRvmdJbpiYAiR8s3V1ddm/97Wv3Yra2llrpGQtAPvrnTtXffzxx588/vjjR4b6PSIScM0jJ6JCbRuYo20D3ec688iDuHweecKfB/05LBhMEQGV7HGv6wMwE+o90MQY28s5r+GcBwF0T8T7YftNN1148sMPFz/b0FB61/TpbYwxvNrcXHQyP7/7CzfeeMHzB4S+pFCNEKJGJ4XWSSk3nDh6dN0KKbt/45prLmYDPeTz5ZwIBgt31dWtb2xszC4vL3eeQBv99oynXcqTIGOsfSjFPqlZ6YuJaCn6Z6U3pkiVnkySndhoB/BrALugFPrrASyHsnu/F0qVXpvkNQHvCtteKd0BbyzcjQLdYDCkA6NRoI9aXhvXWFzi2reV6H2jO0fWqeOmViHEaVeBPMQYC0/Q/j2TC+iZxKjv9wc/+MGK9595Zsui7u6smQDef+cd2vXqq41/9Xd/t3PJkiWRQRdR6vB8unxEVQ6AbCLyu84d1n1LXyuxmyPyxztvuWKmIgy0cS4GUAZlrX5cz/1uHm4vn0pWrlzZmX/llQ3/vm/foizO5dK8vN7jnZ3ZL7a15Zdv23Zq4cKF3SNfJflwzjs557uI6H0p5RIp5cYLFy7czS5csO+/4org0sLCIBjjNzCW91Jt7YoTx4/fUlFR4XddogsDi+RB3RgUHE6xrxssljPG5jDG5pOalV5Nalb6ZG3sTHasYQM4oY9CAGsBbNJ/9yCATwAchhodkGy8VIZ7WUBP6WfbFNANBsNoMAV0g8Fg8IZENoDOhvSyzRwRCdu2i4dI9pRA22hpwq4i+Zm4uXpdI8zVmygytoCOzJthmDB//aMfrSqqrp79/5eVtc0JBGxJhGfb2gp++vzzG/fefnvTpk2bGBEVob9I7lZEAEoRIdFvG9iE/iJ5CCrxM5pmFcc20En0lGBgkdz9OXRsAxsZY8fi7NajRJRr2/Y6KeVGItpu2/YVRPSJEOIoJiDZsHXr1tC5L31p/+u/+MWqdxsbywCgt6Sk89rPf/7opz71qWQoCUYFEfnik2pEVCqlpMaTJ3FtRUWOv6RkoQ2ASRld5vN1fHT0aOj48eM127dvP+OytR/1Z0sn404zxk7r99dyPSt9LhF1aFV6FRKcy+i+NKaGmiIeAlCjj4ehEkPT9J/bARyCsitsT9J66ahA98LC3SSCDAZDOpDod+WgsUNcY3GJbix2x0wFrtNt13iqGqdArt23JrKxeDicBlyWYQ24mRgrJsyBAwcKd/3iF5v+q5T8izNmtHLG0BCLiW/X1s7+6x/9aPXf/vjHJ/T734mZHPetbKg9CgMAl/tWJ5T7Vjv6m4tHZXtO/RbgffGSS93sbvCPot9960JczNQJAFLKhVLKzUS0goiWEtEhIcQnjDHPYxTOOX7nO9+pfOKHPxR/efr0DP+lS0WxnJxo3saNNb/3rW8d9frx6Pg0P64RoYyICuvr662icNi36Morp9nANEYksywrsjQnp2PXRx913XTzzQddtvZjsp3Xv/shqVnpS4hoCZQqfQ0RXRijKt2rmCkVa3QAeA9AHYAvAbgE4CYANwM4BdWAXJPEtdNNgW4s3A0Gw6TDFNANBoPBG4ZVoOtkjwCAaDS6gogK45I9RXHXatPJnjrG2CEn2aMto1M6cytFEClr7UwkIwrooy1k2raNqk8+ueK38vMxu6QkV/p8fgjh+3x5uXjj3Lmco0ePfnbdunWdUEVyS9sGdkMF6G7bwFHZTRORoMstwB03h5FsA51kz4hJV8ZYxGVTvtS27U1Synv07LqDelZ622ge+3i577776m+++eam3bt3lxERtm7d2joe28eRIDVLbzD1SSlUkdYhhv7E2tG21taS3vb2qN+yOmV+foUsKJhHWVnlvqysLsaY5JzXM8aS8ri1umU3Ee1xqdLXulTpJ6CszRO6HFL/WU/19SWUW8OzAOZA2btfDeB6qGTQAajk0HgSEl4X0L1QL3lh4W4U6AaDIR0Y8TtZx0wkpSzr6enZRJePpwq4Tnc3Fp91xUyTubF4SFyPNxPjpky754Tv94033phX0dub/+DcuWHm9xdLIXwzhPB9Voisn9fWXtfb27tYCMGgYu5uAD1IgvsWgLw4C/AS117ebQHejf69vNPoGtLuWyO6OQghaoUQtVLKItu2NxLResuytgCo1TFTlZdq9Dlz5vT85d/+7Ud79+4tuXjxYs7cuXPD69ata0/VzHldJC+Mj5dcz7U7t98O9VzXRyKRuo7m5mKRl3eRZWXl24WFcyk7exrPycn2+f29jLEWzvmlZDxGPbbiFGPsVJwqfV5cA3Ii7zGv8iNeFOlfh3r/r4Zy83oIKlfhNCCPN973soDuVcwEmAK6wWCYRJgCusFgMHiDJCIupXRU5IPN1csCgFgs9hkoG2cn2VPvSvSEOOdjUlVOcjJRVZCpCbC++9WJl0Lqtw0sIKICADm2bWcX5OfPLBCCy/x8G1JKSGkzy7JELBZtaWmpI6JDUEryHhq9baDToDJY4dZ5jBbGYAE+qidD2ZRXCSGqpJSlrqTQNYyx05zzvU1NTeebmpqySkpKYgsWLEipLWBpaWnszjvvvJis6xFRllaO9yV6hlCfDDr7XR9dTmLN7/cjf968xXt27ly2PDu7s6C7ux0tLaf3hMMLOouLZ964YsX1sVhsK2OsUgixj3OeFMv1FKrSk4kXCRT3GvX6eAPACqhi+r1QSaJKqKRQos0F8WsA3hXQ0yLpZBToBoMhTSAAkFLmxKnI+9y3oPdqUso1UspVUI3FIc75ecZYZZz71lRsLB6OTI0fMi1WvGxvoh22Suhy963s6dOnl1/IzS1ghYXZEiBIaUNKO1vKaLijI2bb9kecc8d9K+H9iC7cFgxRuI1vVunE4BbgIcZYUuaCc87bOedvE9GvbdteIaXcZNv2F2zbbuec77Ms62B1dTWXUmLJkiXh3NzclO29OOfYunVrCCoWHTdExGmg01n8cy2cUzF0E3dIF7IBAPPmzcvby9imA3V1tLmiokG0tze0AfmVra1XLL3vvhmWZf02gDrO+T4hxAn3746HOFW604C8ntSs9AtSypOY2LFYXnyXuOOMbgB79DELqpC+BcCnAJyBakA+ibG50KWbAt1YuBsMhkmHKaAbDAaDB1iWVdrT0/MABm6k23Vw0SiEOCalzJdSbs3Ozv6Rz+frmMCHOxFkWlIEyADVOdAXnBRJKSv0/6+yLGshgFwiykK/bSC5bAPbGWPnZyxd2vPqm2/Ou9W2m3N0sXp3OJxda9u9Ny1bdhDAkIVel22gk3xwKyKGsg084ko+BBljnV42q+hGmTeJ6F3btlf29vZu3rdnz0MNhw6Fey9ebOzo6TmfvWxZ4+cffPBsaWnpRBZqBxD3XJfFFclzXacOpj4JavVJZCT1STQaZT/72c8Wffjmm0ubamtnvX7mzIwbKyqCRbm5vXV5ecEl27Z9PH369LBt2+uJaJ1lWZsBnHMlhZJixToOVboXdoReFtAdolDF8oMAyqGSQmsAbAVwASopdFSfl+gaGGSdZON1AT3lCnRS80kNBoNhKiMjkcgWKeXnXD/rcVmt13POQ7FY7EbO+dGcnJw30rCxeDgyuYCe1uhm4FynsRhAsWVZN0PtpbOpf0QV4t23li5deuE/pFz3wdmzPTfk5HQBQI+UeKW5ubxky5YTfr//zDDrcpf7VgkNX7h11M3nGWOH4wq3nu1BGGO2z+c7DOCwlHKmbdubz549e/3hDz/cHqmtbQ63tV14Ky/vwpY776y79tprg149rpGIe64HK5I7EvY+x0HGWM0gTdwjFv727t1b/O/PPHPFqVOnZn3Q1ZV7VVFR+FMVFaGzALqvuGLfLatXH+ecL5FSbpBSfk5KGWGMHRJC7Nex2bjRj7OKMValm5+u0g3I80nNSh9KlT6VLdxHWqNBH28AWA4VN30OyhXiMFTc1DTKddIilnGtg1SvRcr90hTQDQZDQpgCusFgMHiAECLo9/s/YYxVCSGCnPPL1KvRaPSqaDS6NQ2VEomQiQV0hyl/30TkB1CsFRGOkjwParZelj6HA7BjsdgKy7JCQoizjLFaxpjbNnDAZ+Lz999f82effHL71y9eLN0WCMQuWRb/NWOo2Lr1xG233XaRiOJtA/vm7GGgbWAEKtkTYoyd0X9OuHDrNYyxmM/nO/jcs8+2xj74YN0NixdPn7lq1eJgOLzw49ralpeff/7gI1/5yt6xXHvXrl1lr7/++oLOzs6sRYsWBb/whS+cmT179rAzDZ2E3iDP9WAWjc5zHWSMVcP1XDPGxqWe/x+PPba+6Z13Vt4CYHpOTs+HHR0F/3nxYvnqW289eO9999Vs2LChHQAcZYqU8grbtjdKKT8rpex2JYWSMi8xAVX6GSI6iYGq9Kk4Az1+jeHu4RKANwG8DWAZVFLoLgC3ATgGlRQ6n8AaGGGdZOD1DPRUKym8slY0GAyGVEJ+v7+WiPZwzlu1ivwy9WosFruOMRbNsOI5kLkFdCAN7nkY961sIsqBztEyxgqllIU9PT2bOOf1Qoga3YDqxE3dbvetzZs3o2L9evHDDz9c8VFHR8l0n8/+IBoN1JaVtf3hV75SSUS+eOctV8xUjP7n1sbghdsQUyOqJl3BiXPeWFdX9+Yr//APlzYytnDtsmUz2KJFG47X1S07+OabdaWlpW9dddVV7aO9bnNzc+DZZ59dUFVVVZ6Xlxe95ZZb6m6++eYRVdOkxoENNaIq/rlOidPZ7t27S3782GM3Lrt0qeh3c3N7Lggh325tLfqXgoLII1/5yt4bbrihpaSkxIJqcD0qpSy3bXuD44IGZY2/n3N+MlnW+Po99BERcaZmpbtV6Q26Adn9/KZrAd0hBuXYVQmgDMrJay2AzVAF9oMAjkC5tI20TjoV0L2cgW7iJoPBkBCmgG4wGAwewBiLZmVl1UDNiB2KTE6IZGIBfUq93kSUrRMtRURUCKDAsQ10iuT6PNuliGiEax653+/vsW17tZRyi2VZ6wDM5JzvEUKcHyw437RpU/uf/eQn7z337/++5vWTJ2fmFRfzz91yi3XXXXctt217m23bftfpjm1gc5xtYJAxNlLgOelob2/3NR44MOP2QKDpimi0lhoasnKLixdcs2DBol93dd1+4cKFNRUVFR8LIY4nmsz6yU9+svSdJ5+8enEkkrMEoMPvvYc9O3de+f2/+qu3V6xY0UlE+UMoIvpGTGi6MLhFY8qe6/379xed++CDK76Vk9NzY3FxGAA+O21a+w/r66cfuXChZN26dQMSY4wxWwhxXAhxXEpZppNCay3LuhrAWa1KP5lEq0K3Kn2RTgqtJqIVLlW6F5/1iVCgD4YN4IQ+CqESQk5i6BJUUqgSQHiINZDgOuPBkwSNxli4GwwGQ2KQ3+9vAnAKw3+nEfq/xzOJKRU/JJEpEys67ltxRfJ8qJgpGy5Ft8t9qwNqLE4HgDYhRDuAcinlVinlCinlHN0IupdzfllDKucc/+eJJ/Y9+eST1r73319sdXfnL12/PvroZz+bu3Dhwi/FYrEC1+kx9DtCnYgr3E7JUXH79u2r12dOAAAgAElEQVQrK29qyrpx2bJK3tRUKQsKKrbOn7+4va5uZdPFi8uWLFmyX89KT8huvaamJvf73/rWTYG6uhlriFgzgJ++++6K/fv37/vud797zNWQUBYfO0G5BzhYmIDn+tknn1y+orW16E/nzGn2q7nsHdu6urL/LBQqyMvLs3XxvA/O+SXO+RtE9LZt28ullBtt277Xtu0wY+yAEOIA53y8s7oB9DUgD6ZKd2alVxORGOk6SWKyFOlbAbwF4B0AS6EakO8AsAOqAfkggLph1vGqsJ1OCnQTNxkMhoQxBXSDwWDwhkQ2Z84mMVOTQVMiKZJEJlVyQqsY8l3JHqdIng2lJPe7zo1BJXu6oTrFO9CvIg/TEPPIGWPw+Xz7iGiflHKxlHKLlPIeKeWtWq3cAiAPar5fKYCSK6+8Unz/sceAgbaBFxljlS5FhKe2gV4QDAb9orvbP6ukpBsAmGX1+i5dOjXDtk93XLq0pLOzs6ihoeGzL7/00t21hw93sJyc89tvu+3wl770pTOcX/4VUl1dnfv2009vfkjKwBcXLYowvz/QBmT/6YULV7z68suLli1bloWBcww7oJ7rRsbYsTgFSkpdMqSUuHTpUiAvL8/Oy8uzAWDfvn3lRd3dWdfPnt2nTBCMYXtRUdeec+emNTU1Zc2cOXPQ4r1WsL1JRO/Ytn2VlHKjlPLzUsowY+ygTgolZX6hTgpVM8aqpZTTiWgdgKWMsdVaITwLgB+pmZXuVQJltGt0AHgPwPsAFkAlhW4EcBNUgeQggGpcXhRIJwW6VxbuSRlTYDAYDBOIRGLfl5kYOwD6udF7ikxissVNPvTPI3e7bzmNxVyfJ6GsmbuhimQdcDUXY5gcgRCiQQjxgpTyTdu2NxHRRj2e6Dxj7BzUXrKvcMsYy33kkUfwyCOPQK/pFG5rWP94qiBjrGuyuW+Nl45Ll7IW+nw21/fFOzubeGdnU05T08yG+vqStra2dS+//PLVu3fu7Ors6AjNXb78yKOPPnp04cKFg7pj/d2Pf7yq4uzZ2T+YM6drek6Oz/b5sv4jGCx79r337qq9994dc+bMGaoh4ShjrDXuuU7pezccDotwOCzKy8ujnHNYlsWaTp2adW9eXrffFROuy8vrmdnWVlBZWVl+6623DjZqCowxy2WNP103IG+2LOs6xlg153wf5/x0slXpUkofgNUArmSMbdcNJwRgGlIzK93LGGA0a0io2OgUgAKokVjOaKwglJNXJVQzu0O6KdDNDHSDwTDpMAV0g8Fg8AaJkZM8EsjIhAiQuUkwEBHzKomh31tu28BClyIiB/2KCLgUEV1QSvJ29Cd7RqUyJiK/7tIfYGcH1aVPUAX6Vfr0XgBNY533li6Ul5dHrby86PmOjqwZeXl9xda6jo5Ap203vvLKK8fefvrpG7bYdtH9hYWlNc3NZa//0z9d1dbW9sk3v/nNfdAJNed5PnHixIw5gUDRfVdeyUgIEIBC247dHolYf//++zLyjW+8X1BQcMlVJJ+QhoSnnnpq/vM///n6joaGaVIIO2fmzEsbNmyoy8rKsmIARaTkBUL0BdQdlsXJ55PZ2dkjvjd0UugIgCNSymk6KbTJsqxtAGqEEPs451WjSQrR0PPfyzDQ2r4HQKcQogzA54joolalj2bG3Yi3iMmjQB8MAnBGHzkAVkElhR6EcpA4BFVM97qA7oV9X8pVG1JKkwgyGAzpACGBuED/W52JsUOmKtABj+9Zu28VQ42pGtZ9C/2NxRfRXyBv139O+N9/ImJDOEK5Z5HPJaK5UPuXdgD1nPPT6B+bFBrv2KSpRvH06T0NlsVtKSF00VgS4Xxnp9XR3f3J17/4xVhBff3CHfn52cLnm/vOu+/Ofqy2dv1f/M3fvDhz5kwhpexz3rIsqzTc0DDrgQULfGUzZ1bYACClvCs3t/fV06fx/vvv1z/00ENVE92QcO7cuewf/Pmfb6rau3cpWZbPV1LSPnfJkoubNm2qh89nd/b0uBujYRMhAmBWVlZCzZac82bO+WtE9JZt2yu1Kv1+27Y7OecH9NGR6OOly+e/l6G/AaQY/SISG0APY6ybc76D1Kz0WiI6hctnpY8Vr8ZeAWPf/3cC+ADAhwDmQzl53QDVgFwFVUyvhreFba9iJiD1CnQBEzcZDIYEMQV0g8Fg8IZECsSZnBDJxAJ6Sl5vHQwU64RPoS6S50IVqXNc65G2We8BEAJwDv2JnhBGGaASURYRleoERAkNLJIP1aV/zFFEAOiSUi4hos0A5un5aPWc89pkdbkPhZQSzz///JyPdu5cGG5uLiyYNavtUzt21N5zzz2NqVx3OGpra/Na/X77pdOnp0sp2dLS0u7GcDjwXjCYl7tp0/nXXnxxy53RaPYfzJ7dwfz+AAKBnKcvXQo89/bb21offnhbcXExAFB3d3dbOBxu6+zsbLEiER4IBoPCsqIsFouCiERHR24ngGg0ulcI0acsr66uzn366aeXtLe3Z69YsaLlwQcfrMvOzk7p6/DMM8/M/dlf/uVt23t6spZxbr/W0VHeWl8/68CBAytkQUFnK+f2k5zLr86YEfRzjsZo1PefXV25s7dvPxpvRTgSnPMWzvnr2qpwhU4KfcG27U6XKr0dGLZIXgog13XZCIBWrTw5DaA13tqeiIqI6CrG2DzG2Gwi6tJJofhZ6WNhshfQ3XQD2KuPWVBJoc0ArgPgqGJS7caSjgp0kwgyGAxTnYQK6DAW7hkXNyW7yVw7ZuXRwBFV+TpeSsR9y4mZBhtHM9y6XO8HS53mYvQXEkswME/ruG81OI5QUErUfCnlBiK6Uj/WTiHEeWfvmkqqqqry/v3ZZxefr6yczYWQS7ZsOffwww/XVFRUpNShaija29t9XV1dvoOxmMg5dmzGDQsWBDlj2N/UlN9QVtbVduFCme/cubl/M316ZHZubi/5/dl3A9n/7fz5Ge++++437rvvPgCAlLI7FAq1+Xy+1nBnZz7r7rZFU1MHi8V6Yds2J0Kss7O8sbHxmM/nO+asH41G2bPPPjuvsrKyoqCgoPcL/4+9L4+Pqjrff885d5bMkslGSIAshLCEPZDIviqoiFAKCu5atbVa9dvNn9attX6t1tZu39ZWbV1RFBWFsoisIoQ9QMJOEgIJWWeSyewz95z398e9EyZDAglZWvE+n898/Kg399y5s533ed7neRcvLsnJyXG3ecFdAK/XSx/9/vevN506lf6QThfc73KZj1RX964+enTQmg0bvNUxMb7ljPFxVqs302gMcURYVl8fV2+1eh+45ppzHVmLEBKSJKkQAAqFEKmc87FCiIlCiKmEkBOqK72EEIIdFMkb1JrpuForhZ37TeFzAUA2ImYDQC4ijkBlLNYR6Lwr/b9hBnp7gQBwWn2sBaUBORcAbgHF6CBBy1Fr3YWedqD3xCgvrW7SoEFDu6AJ6Bo0aNDQM9Ai3C+Ob52A3pk4N0TUw4UieZjsMYB6L9XYwDDZUwtKJ7Mz4tERpy0AgClCSAwXxvFwoZDoA6UodgBAeVT8d5td+oyxOkTcKYQYxDkfL4RYKISYTSndwxjbRwjxduA2tRuvvfZadtGHH46ZyDmVAGDNiRPpv9u6dfimTZsOvvzyy7v0en2PxUZ6PB72m+eeG127d2+WMRCwlHEec7ypKTN/xIhQQlqa1Pe66wy9U1Mnbfn4Yzo/MxPAZALkPEQ4D80xmZzvVlUZDh065MzLy0tevWoV7l2/Xnjr6tDNmPuc12tZV1mJN8bHBwAAgkLAF263KWH06FPJycnNpNebb77Z//Xf/362xe02JxGCu1eswI+XLq147e23/92vXz9/dz335e++O2aK3294Ki7O+atz51JnCUGvliR5txBUlmVpKefGv+t0orCyUkoGwFOUIs/OPvfUQw8VX+6aKil0ABGPcc4HCiFGI+IkWZangtJcQqAlKXFJkfwS6zkJITvx/Kz0QaDMSh+O52elX64rvafcFF39eTinPtYDwFAAmKT+94cA4BAoDouudOqH0dMCek/MQNci3DVo0PBNR3u/K791tYOKb62Afll/1Hr6lhnONxZHp2/54Xz6VthJ3gAdT9+SooTEZqEcWgqJAgAa1X1kGSFkX0TcesNF0rdqGGMlQog4Nd59jCzLEwghxyilOymlZ7rDFV1WVhbzytNPT0o7ezZ5gdHo39HYaF1/8ODAdStXjnzxlVfWjR07ttsF/EisXr06ZcXrr481OhwJQVmOWQag3y2EPGzYMIyZMkU/b+pU07tvvmmcarFAau/eIITgRJZDvTj35ssy2/3VV/6bb75Zf+DAAcu6Tz+Va48cEZ5gEM55PHUr7fa4qyXJF0MpAgBsdblMNQZD8KYJE6rD69fU1Ojvv+uuGxpOnszoJwRtAIAvPvpo4u0PPbThoYceOtldz3vZsmUZ3tLSvv+IjW3a0dRkNQeDxpcY4w1CkGohaKHXa1rJGP9xfX3sEADqQCTnLBbvtFtu2Tt06NDLFvcppVWEkDVCiL1CiFxEHMI5v41zLoPSCNw8vgDaIZJfbK2oWek2ABiqzkpP6wJX+jdJQI+EHwD2qI8UUJK88tRHL1BqpiPQPfXAFRPhLstyeA1NQNegQUO7oAnoGjRo0NAzuGTMYITL9ttGiAB8e0kwgFaed1isjiB7bNDSEaGPOFYmhPgR0QuKI6EJFKG8Qf1nu6HGBlrbcEQkQMsZ2W5QHBFhIbFLYgMJIcgYO84YOy6E6M05HyeEmKp2uRcxxnZSSlud23Y5qK2t1ReuWZNzE6XiXCjE/lZWlpHAOU0Rgn7xwQdXb/vii9xh2dkVqYMHVy9avPjkVVdd1dhVayMiw6hY+1WrVqXpjx9P/Fl2tm5YYiI4/H5YVlKCJ0Ih/6Lbbqvs1atX/alTp7xut3u00+v1SwZDs+vFGwoxn99vcTgcq3/zy18OjCkuHnlLSoqtd2Zm4sHqat97khR43uWSDvj9hlTG+F5Z1pUkJDgfuueeQ+FzVFRUGF9/5ZXZcz0e09OxsS4rpVgUCkkPl5RkPPPUU+P/9dZbW7rq+UfC6/XShoqK5JlGo3+H221hsizdzVjISimcQKR9dDrZRQi8xVgoZcmSArfbbZicldW4YMGCCpvN1i6SABENQojEiPd1IrbtJPeDEjNuBAAvIeQwY2wXpdTeFc9XJY1KCCElquMpTAr1Q0RXBCnUUTdPT8WedwdCoMz1kwFgESiR7sNBcaZXgUIKFYPy2nQFelJA74kId20GugYNGq4EtNeB/q2McI8Qnb51zx3aeM6tpG+F55GbENEY8XeIiOF55OH0rUiRvKPpW/rofXyEi9wWcagMSp3UAABhIdERISRetlBEKW2klH6JiFs55yOFEOM45/dwzqsppTsZY8VdOQLrk48/zkw5e7bXnUlJDY8cPpxt93jMmQBw3OGIWzx/fkZOVtaZPunpdWOmTDl9zz33lHZVEzIiAkZF29fW1vb6+rPP0udIkvGGceOoSZJgV3U1LGtokM3DhztuuOGGM5IkORqdzlTmcvWRzp2rB6W5HAAAGr3eeEdj46kNGzYc/Perr14/FbHXHX379nX6/SkrSko8XyLKPzx3LmmCXh+yc053UArpM2YUTZkypbkW+NWvfpWPx471f8tk8o3V60NeIeBFt9v63l//es3MmTOrusuJfuTIkaT+iDRDkvjfPB7TbELEeMb4CQDm4Fz6SWJi/VG32yqNHl3q79fPlWQyhW6ZNatizJgx7WpwiEhJaDGeqg0neSMonzGr+s9ySuk+SmkxVZsPOgu1AblAbUDuSld6Twjo3YVqAFgDAINBaUQ2AsACAJgDAEWg1E1dmabXUwJ6t0e4h0IhTUDXoEFDh6AJ6Bo0aNDQM+CgRbi3CZUM+lY9b1RVciFEP1BEcSsos/XC88iliGODoJA9XlCcmJGxgT71VO1dNzI2sNlBfonYwEpCSFGUI6LbY/oopTWU0pWozF4bK4TIl2U5FwBOq0L6ic44+QEADh06ZNM7naZkk8n/7LFj2d8Tgjyq0wXfCoV0cZzrEu32pEwhWFVlZfIfDhzIvP/JJzfPnDmzvr3nVx0o0eRaIiiibQtyTZZlx8EtW3SzKW0aTYiTVFf7U0Mh/52MyT8pKoovKCjYunDhwsrBgweDPiUl/t0TJ9KHJyV5YxnDgBDwWmNjrEhKaoiPjw84CguTf2EylY8MBv0iFEoY3L9/CjIW94bdzpc3NARCTieXEhPtC+fPPxD5fJYtW9Y/xu02P2G1uqwq6TFCp5Pv0umCL+/ZM8Tlcm2zWq1dXmwajUYhxcQEKp1Oi0WWWSIAWCmFICL4AcBAKQ7Q6YL6QMDw4IMPHrdYLK2Sja2I5JH3O1okb9EAQikNuyKaXUac8z5CiDxEHC3Lch4h5LgaVVja2fdeGCp5GulKHwjnXenViHgUEasveaJvVoT7pdYAANgMAF8CwEBQHBZzAOBaUJwV+wGgvJPr9MiMPRU9EeGuRRFq0KDhSoAALcL9Yvi21osIAGYhxCA4L5KHR1RFp2/54XzN5AKlZmqEDqZvqeeLiUjfih5RZY44NADnR1SdjaiZwjOyu3UPQAgJSpK0FxH3CSGyhBDjhRDfEULMIoTsZYztpZR2WsytOHKk99UGQ+il0tJ+Bo/HvFKSQggAz3FuDPr9xn4nT/bv5XbH7Dh2LO3U8eOJL/3ud7spbd/HFNue/95aI7drz549PlNlpbw4Pb1C39joI6GQf0YwGDhWVxe/dc2aioULF34FADB81KiszzZuvHar262fZjb7AQB2+XyG7YTg1ZMnl33x2WcDR9bVyfempR0iHo+R22y9++fm9qotLmZFsuzbU1UlOGP+/qNHn/jZY48Vhp+PLMvk0PbtQ38oSXysXh8CADBRCo9bLK71LlfsRx99NODZZ5892Nl73hpsNlvgNADYZZnKiDRdrdlcAKCnVKTr9TwOkRgtltCvfvWrA23c79ZE8nDd1FrcuiPCSR6uoZqd5IhoVJs48oQQ3xVCTKeU7mOMHeiqFLmLuNLDDcgliHgCLj0W65swA70j69QAwBZQuJxcABgNiiu9GgAKQUn06mwDcrc3A6vo9vumOdA1aNDQUWgCugYNGjT0DNrjkggXH99WMuiKI4LU19IW4SQPOyJiEDEJACAYDM5njDUQQppAEcmbAKAClBq4UX10SKzGC2MDIwXy6NjABpXkCccGhovixq50LHQGhBCvJEnbEHE753yo6q5YwjlvoJTuZowVtic+uzXExMRwwZhYUVOTkMQ5/R+dLngOkezknD0AIJIZE6f9fuNPBg0q/1VlZfKHb789Yvr06ZsjySBE1EXf6wiyJzZiucj578VRUXbuxsZGXW1p6dwUxgRrbGwmGpL0eogVgjocDgMAAKUUHv75z7e/9ItfWG+ur48fCgAlAFATG+u+50c/2lZRUWGx+P364cnJTgAA6nI5aFOTg548OUA4nXE/zskxZg8cSHbW1Ehfff75lA/79nUvXry4AgCgqanJYAYAWxTRl8qYQFmWnE6nrjsEdEopjJ469egnn3024TsAWAuAFUKQciGYYIxnGY3+VU5nLKSk1JjNZp0QIkEI0TxXDxEThRAJ5eXlpurqajAYDDBgwACvzWZrLSXBQQhpF4nAGDvHGFuJiF+ESSHO+e3qey9MCnVo9mVbaMuVjojXtNOVfqUJ6AjKd9Rx9WEFgFGgEEMjQUncKATFqX45pPAVFeGOiEQIoTnQNWjQ8E1HewVixC6eif0NQbhevKKeu9oMHIMXziM3gZIIZOacjwwEAmmU0gZKqRcRfaDsBdxw3kXe0fStC5zNoOwtwyOqjBGHh5svHYSQEohI3wIlrahzN6ELoCZ5lajx7omc83GIOFGW5SmEkGJK6S7GWIdmYEdCiokJVgUCUmFDQ/yvKeVZlOLLwaAuHQAeZUzeKgQdYTCExun1wZcKCgZ99dVXpdOnT29u1FVF8tiLiOQXa+SOHAkWOnz48IjYhoZRxtjYFslk/YzG4FcOhyX873feeWfZ7oKCI0/u3JkzxOMxMQA4whjvlZd37N577z314MKFw0aZzT5KCEAw6Jfq6sqZ02knpaUDh8bH6+8fPZrVe736VadPD/3lE0/AX1577Uu9Xo+BQIDKgYA+hdIWdZGJUogDQJfLFSn4dykWLVpUuumTT6563eWyMEr5UVmWEgDgJCIZYDJ5i/1+nZ0xPnPwYLsQItz40SKFq6mpKe7UqVM0EAhAamoqz8jIaFBrpOiUBGd7GkAIIX5JknYj4m4hRLoQYqwQYqYQYiYh5KjagFzeVZ+TKFf6ALUBeXQ7Xenf1Aj3ttYJr9EAAJtAEdMHgNKAfC0AzAalAbkQlFnql3NNV0yEeygUimwO0aBBg4ZLQhPQNWjQoKFn0J4NoBbh/g0EIkqgOBJs0FIkN6qOCKoeF3ZE+ECZoXyaUpoqhBgshEgDgHLG2CFK6cn2FKmoxAa2mKvXhmgbjg10QBfHBvY0CCFCkqRiACjmnPcVQowTQswSQswghBxQI7YdHTlnfn5+wwf9+tkPVlb2jgcAHSFQxDnTAdCJlMqVhKDMOWOSxGYlJclHamqyGxoaMDY21hZxvy0Rp4x2oETe7zbnvwMAJCQkhPS9ejUdKi/vdZXN1iygn/B4DA06ndy/f/9mUnDatGn2jKVLP126dGl2RUWFrbfF4s/t08fV2NhoqKuri/FQirXBoJRiMMgAALUuV8zZhgZThk7nXULIWQNjceMHDUrSlZf3KSwouGXBggWbdTrdobFjx9Zsev99+atAQD/daAwCAAhEWOP3G2IzMir79OnTbTPQf/GLXxQ+dPp0wptFRQMChEClLOvHMCby4uPFm4FAr21WK739oYfSQqHQ4xF/5gUARyAQaPjgnXea6nftMsa5XHJTKBRYHhvrHHnttacWLVpUYTabO1UgE0ICkiTtQcQ9Qog0lRSaEUUKne5CUuhyXOnf1Bnora0BrazjAoCv1UcGKKTQNACYCQAnQCGFTkL7CZduJ2gi0O2uDc55t4v0GjRo0NADCEe4XwoCNAf6NwqqSG6NEMkt6r+Ho9bbSt+qlSTpLOd8IOc8k3NuJoTskSRpT3uaGFXR1hZdL0U0F+siDm8CZR9fQwg5GiXaXlaz7n8KlFI7pXQNIm7inOcKIa7inI/knFeo8e5HO1oHjhw/vnJbYWEWF4L0plTIiHCYc7YYAGyMCcI5C3HOxick8H5VVbElJSXjJ02aFIKW9zs8dx5BaRR3EELKCSGFUff7onv3tLS0pn2EoCMUYgk6HQdQapYir9cYN2LE2fBxer0e//aPf3z18ccfl+7YsaOfn3MyLiWlKTEx0f/RRx9loMEQPOd2R6aCweFz5+KDoRDeajCUT+EcRVJSYk5cnO1/a2rG79mzJ27ChAlbzWZzTa/09Kp1J070/05MjJ+pNcCeYFBXSSnemJtb05F72xHk5OS4b/vRjza9/7e/TQ/4fDH7EekuzulIszlUYzSaVvj9tphRo/jdd9+9RJblC5zk+/fvr9i4dKkjpqoK9KFQcD0hXhg4sObu73//yMCBAzvlFieEAGPsDGPsDCKu45yPFkKM5ZzfzTmvVxuQD5JOjHyLWk8AwElCyEm8cFZ6Uxuu9CtVQA9DgFITnQQlKWMUKHXTCFBE9nADckcajnqqzuixCHfEDsQ4atCg4VsNTUDXoEGDhp5Bux3o7TjuSsR/dQwjIhpUsicuwhERdpJHziPnoJA9PlAis1wA4AyFQs5PPvkkds+ePSkGg0GeN29e2fjx4xskSSpExLVCiCGc84mc81s453ZKaQGl9CAhRBcRG9gich0uHhsYJh4chBDXf4MjoqvBGKtkjH0qhPiSc56HiHmyLF9FCDlJKd1FKS1pz/PW6/V4+4MPFj53+nSv8sbGuG2hkMELQGRCAA0GejYUool9+oCcnj5KrqkBm04HkiQlIKKdEOKQZbls9erVbPOmTWaf3+/Pzc09/b3vfa/kcgRbSZJw8pw5Jza//nqiVFmZkG+zeaoDAd0ql8tsGDnydKSDAwAgMzPT9+STTxZt37494eO//W1MY2Fhpg4A3ABYGgjo/15ZmfRA3771yXq9XGC3x+7inM3p1as+BoBDU5OdOp32PMTE7adOxbnd7jkWi+Wa66677uBHS5dWPr5nT8biUEiXzhjfGAzqt+j1wfvuuqvdMYztgfq5ShBCJABAosViSfjXu+8mFBUVWbdv3y5tXrcONtbWsm2cG4O9eoWmLFhQfdNNN51U3REtnOTvvfde/4bVq0fek5xcl96nT/Dlw4f7HSwuHrX9669zX335ZWdsnz72vLy803fcccfxnJycy3aMq6TQWcbYWdWVPkolhe5SP7thV3pXkUIXc6W7VVf6MTjvSv9GOdBramr0wWCQ9u7dOxAxJ7M9hFO5+lgLypz0MQCwBBQX2gFQiKFLNdP0tAO9u2egU82BrkGDhisAHYlwv/I2uJfGf3W9GJW+FY5at8D5eeT0/KEYAKVmcoKSvtUEAI1HjhyRP/roozSn02kcOnRo3a233lpuNps5pbRACBHPOR+PiBNDodBkQshBxlgBIaQhKn0rekRVpCgTLdpGjqi64n5HVWdwASLuFEIM5pyPE0IsEkI0UUr3MMb2tXffevPNN589Xlx8xFdRkbosGDQMoBSBUhKiFM4xpkfGICknJyNotRKdzwfJycmJiNio3t/SkpISz7JlyyxnystpfEJC/a233np07Nix7ZrLHY25c+ee27F2be3vSktT5sTFuWMliX/V2Gg9YLV6b5s3ryTyWEmScMmSJWfnzZtX+bvf/Gakq6BggCUQ0DkBwBUM6la5XKashobApLg4T0MgwP7d2BiHen1witXqJX6/YH6/ZwhjUmxjY9rZs2cz8/PzcwDgzA8eeaT0hZ/9LPUBp8yy8cYAACAASURBVNN2rV4fqOKcLeOcJeTklC5evPhs61feceCFI9gS77rrroS5c+ead+zYYVy9ejXdXVwMRwIBQ5AxnjB2rPuZJ588pdfrq6Od5BUVFcYVf/3rxBlNTcG5ffs27LHbLa8UF/cv2bt32JcrVozXJSU1DMrJqZg+ffrp2267rVySLl82IIT41PdegRAiU413nyWEuJoQclhtQK7oQVf6UQCITCz4xgjoPp+P1tbWGqxWq5yQkBDdDHCxNTwAsEN9pIOS5DUFAGYAwClQxmKdgEuL4z3tQO9OAV0be6VBg4YOQRPQNWjQoKFn0BEB/b9WSO5G/EdJMLX51NxKbGAMKCK5LuLYEJwXyetAFclB6ea9QJhzOp3S9++5Z3Z9cfGATM6pC4A89vnn42bfcsvXTz75ZJFKKnkopfuEEPUAkCWEmCuEmBt1Kg8oZE9DVGygAwB8V6JI3h5QSl2U0s2IuI1zPlwIMV6N2K5XYwoPEkJCAM2vsym6KWHMmDEJ//roo4QHvvc99uOjR2EupVDp98NLgQAdbTCICX37NjirqjwrT5+O0U2eXGyz2bYBAASDQfLA/ffPrNy9e+hoWWY2QmDVtm1Dtn75Zdk/3313LaUUjx07ZkVEyM7O9kQVu63illtuOSPLMl2/evXgL+x2K9fp5L4zZx7+xaOPFkuSdEEh6fV66advvDEyv6bGdnt6ep2JMawLBKQXfD7Des7pyfp6m1GWaSXntNZo9M2Njz/faU4InHO5gk7Oz+r1+vWEkFwAGPu31183vfHGG/aly5eDu7ERE7Kyzsy/9tojeXl59UII6IiIHi2SRzh/EqGVmeSMMUdubm5Jbm6u/eGHH3bs379frq6uphMnTqxPTExsk9Q8WVDQ7zqjMZhttQaeKizM3Fde3u8+RFwTDOrtPl+Krbo6paS4eNCjn3121eQlS7Y/88wzRe1+Em1AJYV2qoRkhkoKXRNBCu2jlJ7pZlf6CEQchojVQogYuPTMv05fBnQBoVFdXW3YvHZtH39ZWTwNhSgmJXmGTplSfdVVVzVAx4gTPwDsVR8poJBCeQAwGRSBfT8AHIXW78sVFeEuhCCgkUEaNGj45qPdEe7tOOZKxH9cQI9K34oUyS+WvlUPLeeRN0Irv1mvv/561rt/+cs1iR5PTDIAfLhqFa788MPKv7zxxpqMjAwZAHSU0jIhhAcRByHiKFmWx0SdJnJm88lW4qi/lWktarz7McbYMSFEb1VIny6EmEYIOaQmeTWLi4jIIl37iJhAKU14+rnnEtIHDIj55+9/T3yhEEgAsMzng0ZZxqy+fX2xnNs/O3xYOg3gvykj43O9Xu8EAPj444/7/uX5569PcbksIwDgMED2TzdsGPHAE0+su/nmm8+eOnXKbLfb9QkJCcGBAwd6LlVvJCQkhH78y1/u+Nerrw7/85Ejfanfr9dlZVXPv+mmIzNnzqxv7W8++uijdHnnzgGPJiY2DbZaAxwR/l1VFfd7v1/3e59P/1ZTkzlIqThoMAQWm0y+RJ2u+b3iDQZ5ndfrcjgcWxljEuc8f/bs2dNjX3vN+/pf/+p+uqiIsZgY75D8/ONLliw52tjYKCUlJbV7X96aSA6tN4EIUHmAxMTEE/PmzXPMnz/fbrfbm3bu3Cn169fPO2bMmKa21tmxY0dSgsNhvTEzs+6Y02l8du/enNGBgCGfc/i302mzNjYmBEtLMz/dsiV3+XvvnX39nXdWpaamdmicXDTUBuTTjLHTiGhWXel5nPNRnPNaSulextihrkp5uJQrXQjR2BXrXOoy1H92qs4QQsDmzZt7le/encKamgzcYJDjcnLs11x/fVVsbKwMHavNzqiPdaA0IOcCwGJQeKaDoDQgt/rZ6eA6nUG3J4TJsky+rb8DGjRouDxoAroGDRo09AzCboo2EbGJ+1aSQdjNs/zU88diy3nkFlAE8hg4HycHqiPCD4qbsRpakj0dirB+6aWXRvmKigb/PTY2NNZqFSFKpbcbG+PeWb163tHvfvfG7OzsC2IDQSlsbADQBwCAEFLEGNtOKW2roPnWgxAiS5J0QAhxABEHc84nCCFuEEJcC8rrJkC5p4aIP3ODQkDUJSUlHf/Tq6+6fvCDH4z4a1FR30AoJB2XJPlUTIzrUHm57wgh1JeefubJBx7YH/7jt956K6ty9+6hrxgMgalxcQEAgKJgUPfAkSNZzzzzTN4gm82vr601AyLZk5DgHThjRsXs2bNbzOmLBqUU7rrrrtOLFy8+c+bMmZi4uLhQcnJym6RFQUFBonTuXPyCPn0aTIwhAEAvg0FekpLS4OScXrVoUZEQglyXnOx7/aWXpr1lt8c9lJzcEMuYOOz16lf6/YaBM2cetlgsDQCwCRG3UkqHPvzww1f96Ec/6nfw4EH3p++8Yy1Zv370n9euHa5LS7Mvue++A9OmTbOHr+EiInl0UkJ4fEF4dqQ92kkejfz8/IvdLgBQimDu8+nj9Hq5wuvV7aisTPkfxvgZzgkTQnqWUj6aUjwjBNnq8ZiWv//+lM1TppybMWOG/ZInbwdUUqicMVaOiGsjSKGRnPO6CFKoq+LvKSI6EbEMEf2IOAARx4LSkOAihIyMcqV3NTpFnvh8Prr2ww8zU0tLE/OSk10xFos4UVNjOfj551lGo/HkyJEjL5c4qQbFkf4lAOSAQgotAIA5AFAEipheFXF8T/7W9lSEuyaga9Cg4ZuOS9ZMEcd9W5uOAbr5NywqfcsKStR65Iiq8HEClCZIHwDUgFLLOEHZezdBB377ysrKYt7/29+uWRQIWH6amhqI0emkk7Ks/3/V1TlL33tv0M8feyxypnQQlJrpBCi8Zm9QRljVUkq3M8aKNYGkbVBKayilK4UQWzjnExFxpNqI0ATKa2kEpW4Ko3kkGGPs6L333uuora1N+ujdd3M8jY1mRgieNhgCM2W5cenhw6TMbHaPu+mmfSNHjnQCKE3Hr//xj9Omu1zm3yYkNBgIgSAiPO5wxL3+xz9Od9XV7eSnTiXqfD5dyGgMfT1woH3+LbecvpQAPXjwYM9Lf/zjrsrKSoPP52OZmZm+1hqOwzi6a1ffaZLEB1utAQAARgjcmJrauM/v1wdnzDienp7eZDab5YT9+5MPr1qVV+jxGEaZTAGvEOQftbXxjXFxTXPnzj3DGPMzxo4IIXpNmDAhf/z48aOampp0H37wQXXxl19mfvjcc32WWiy+nGnTTj7y6KPHjEajAOiwSN5ACLFHNIHYI5pALniOycnJMG/evEu+9l6vV7Iggp5SXFpWlpIWCBieMhi8tzc12eYish/odIEGWZZOC0HfPHEi8+c/+9mU95Yu3XjJE7cThBCPJEnbEXGHECJLbUC+XggxixBSrNZN57piLUQEROSIeA4RA4iYBQCZoI5fI4SMRcT90NKV3tXo1P7/q6++SqpeuzZrksEgp8fHex1+v27vjh1pq71eafFdd5VRSi+nSTcAAPvURzIoSV65ADARAM6CUjMdhgtj73tCQO+JCHeq/T5o0KChI9AEdA0aNGjoGbRnA/gfdxT8B9ElLhJEZKAQPZGxgWZQYgObHRHKoegHRQxvAKVQcMJ5J3mH4vvUDv3I2MCwuzm+qqQk4bs2G8lNTQUBABLn8t1JSfKaigr4/PPPSx577LEDqqv8gthARIzhnI8VQoyTZXm0Gk++oytnLX8TgcosQ0sr9zss2kYSbAIUtzMBhVQrJoScoJQ2EEKaBUZZlslPHnlkjvfYseS7JSmUpNMFPpdlabMsS77c3JK8vLxzixYtKo8Us7dv25YxlnM2NSamuVt+hF4fGi8E7N66ddgtkycfGZ+R0cgoxQPV1Zbta9b0L+zVy5+bm9umKyAMo9EoBg0a1JxosHfvXtuf/vSnsZWVlYk2m81z6623HrjpppvO+Xw+RjlnNnX2XxhxOp2sk2XD+PHj67Ozs70AAIFA4Ot3//jHKXtqauJtAKRWkmTTyJGlP/7JT5rd2IQQLklSEQAUHTx4MPvDP//5+jyPJ236oEEg+/2udWfP9l7x+usDMjMzHampqbHQBSJ5ZyFJEsZlZdkP7d2b4QqFGJVlaarB4L/P57POAMBZksQ5ANRzTu8xmz373W7L+++/P3TGjBnbuvpaCCFeSZJ2qFGF/VVS6NpIUohSWnmpz69KsMUhYqL6aCbZQCE0wycIgRpDCgCCUoqEkBGIOBQRa1GZlV7V1jqX8xShk4RGUVGRzXDmTNy1mZkOi14vAACSzWanp6Qk4fDevYkRAvrlQgZFMC8ChYzMBYDRoDjTa0AhhYqgB0TtCHQ76aQJ6Bo0aLhCEJ6BrjnQW0eX1IutpG+FRfJLpW/VQ0uR3H0564bToMLCISIm7N+/v1cSov6RAQPAwJhVCCGyOZdv8nrlv6xeHXz4kUfWms1mu5q+5YncSyEiCCGyOOcThRALhBBXq0lU+75ps8u7Goioi6yZohpdYyMO5QAQo/43PwAUU0oPUUqrCSHuSNH2/fffT1/53nsTxvr90jSTyXtMCLpWCPp5bKxnxsyZRx+YObNi6tSpzY2qmzZtSpbr6hLus1o9BvV10xMC91ut7rU1NYneTZsG3j5iRGW/tDTnOZdLv/HQodTVOp2464EHWkSxt4W+ffs2v8ayLMOLL744Ytu2bdlCCDp69Ojyxx9//EBiYqIc8ngMVklqsVeihICVEAjFxgbuvPPOcgCAmTNn1jxeWRn3ZHFxRm+n09oEgE1xca7FDz20PT09vbmeoZTWqXPmN77+6qvz+M6dA3/Yu7eub9++/iM1Nd5VGzakLUtOHnXb7beTrhDJuwJZWVmuL/V6uczt1p92Oq0zKBWrQiGDBZH9gFI5lRAIUQpDKIUbKA395cCBbKfTucVms3XpHlNNRChhjJUIIaxCiFx1LFYu57xKFdKLI+v11hCRMNeiXoqomyJ5AScotelZQohECIkFgNmozEovUxuQuyrNq9MO9GAwSMr27k3O1+n4mD59XAAASWazbNHr+WcnTiSWlZXVDBgwoLM1Ri0ojvQvAWAIKGL6fAC4HgCKQambKqHn6qZuTwiTZVmrmTRo0NAhaAK6Bg0aNPQM2i2gozI77tuGdpNgKqETHyWSm0Ahewzh86iOiDDZUwMKwRMme5zQwU7dCPIhvg3yIXz9HNQOfULIybqqqkFxfj9j9fVuwrkMiMAAwBoIxJ09e7aSMXairTXViOivEbGAcz5CCDFBnbVcRSktYIwdvlK7Z1WRPPYiInnkHsYJyj2vJIQURcQ0NhBCQoioV2dVjxNCzASAoYi4Uy3KOQDAhx9+mFZ56NCgf+j1gWkGQwgA4EEhYLHLZfW6XMYHH3zwZPQ1cllmMa1cuxeR9A0GddMyMhqpShJd1bev69zJk4mHCwsT2iOgR2LZsmX9/vj00wvS3W7L1QBQCgB/2r8/Z8+ePVt++MMfHtlotXoL7HbLtF69muPZdzkcVpqRUZeZmdk803D+/PlVeXl5n//73//u19TUZLh68OCGOXPmVIedGlFO8oSdBQXZ/ex2813DhslUknRgs8Vm9uoFLx4/Djt37rQuWLCgjFIaHidg706R/FKYdeONpz8pLU2qrKhI8ANAUTCo9wpBUwlBBoBeRAqEgEmSMAUADrtcrb10XQaVFCpljJUKISwRpNBoznm1KqQXg/K91SySg+rgB4VgC3+nNLt+AOBwFMHmihbjVRI8hxCSjogzEdGLiCVd5ErvtBDc0NCgTxSChMXzMPpYLIHjtbXmrlgjcjkA2AQAmwEgGxRS6FoAmA3K70L4t6cnZsd363e1JqBr0KDhCoEW4X4RRIhql3zu2DJ9K1bdV18qfcsDnUzfitjDx7eyh4+HloKWCxQRsa6mpsZJ/f5Ma0ODkwohEyEEAECi32/0+v3E4/EcjY2NbVUMV9OAwvuuZDWJ6mo1nnw/Y2wnpfSyZm1/ExC9h4+qUy0RhwZBbW4FgLNR8fZuQghwzvsKIcYj4lAhxCBELGSM7Vb/BoQQ8K9XX5003eczvhob65LUfejbXq/x+XPnkm+88ca10XPNZVkmgAjGKFFYRoRYWZZGWCz+frGxQQCAPlZrcErv3k2fnjiReObMmcpIwfpS8Pv9ZP7cuQvw6NGBo4SgBgDYf/jwwO9s2TLso5UrP0gePLi2cOPGnKuFaNJTigAAZ7xe3WnG+LTs7OZrjo+Pl1/91782r1+/vndRUVGi1WoNzpkzpyJ8Lep73Bbev5eWlqbYi4rSf9C7txjRqxcCIca+NptRqqyEf+/aFef+zncqLRZL5DgBuzqaqcfr+IkTJzq2ffll5Zt792Z6hSCnOGdxQhALIZhEqUBQ7MkGSnmaJHFdKKSrqakx2Gw2b3ddkzqS7StE3CaEyFYbkOcKIa5VxwvsJYQ4wyI5nE88C4vkxojTuUB5j5+L4AXsbZgVCACEZ6WPxPNjsY6CUid0Bp0Wgl0ulwRNTca+amJCGH2s1qChutrqcDj0XSCgh8FBcZ0fBoA4UJqPc0GpnWpB+b3oCQ2pR2agX6kcmgYNGroHmoCuQYMGDT2D9mzQvtUR7nBe+AZQiJ1oR0RYJG8mXRBRJoT4ENEHilASOY/cBR0EIhqj5mPHX4R8CAta4cIsPGvPFdkxHt+vH19XUTHiVqNR1qsEw/FgUDpKCC4cMaJdhZnqCj6AiAeEEANUUui7Qohr/tvdFS6Xi/39738fXFZWFp+WluZ84IEHjsXHx8sAzS7baJE8UkAME3sICoHnIISUE0IKIwiIxuhiOBqEkKAkSXsQca96/8YJIb6juoL3Msb27tixIy1TCDpFr2/uOjdTCt/R6eSXDx/Oau28uXl5FV/u2ZNzIhiUBun1MgBAlSzTPZzTa3v1ctAoYTPBYJDPNjYaWjvXxfDXl1++ZrLLZfmdxeKOpxRlRPiNx2Ne+vHHU+67776j/adPP7V8zZph586cSUyNiQkedbuNhWazf+b8+SejYwz79OmD999/v1cIYQSAbES8KhgMhsmHFk7yxqqq0FCdLiB5PE0kGPRDMOhnOp0+g/O+9tpaCwAMREQ3Y6xSdQP9x5Cfn98Ijz66d/3KlZlNtbUJf3Y6bXpJChZyrr9RCFKFSI1Go78eEY8TgllZWT0yDkElZhghpEL9rsoCgDSVFJobcWjzPENCyPEokbypIy4U9TtoNyLuIYRkIuJgUGalD0fE6k660jtN0lgsltBZQsAvy8QY8f6s83r1hr59m0AhwbqaNEEAOKk+zAAwCpSYQgoAj4Ay8+8AKIJBd6DbRXpVZ9AEdA0aNHzToUW4XxwtBHQ8n74Vnkcei+ej1mOg9fStRuh8+lY4KSe8h4+HliJ55B4+3Oh6lhByKLrRNXzOzMzM3ssBkr9yuciMmBjlRw0R1vj9RltW1tnevXu3q9ahlNZSSj8XQmzknF+FiPmyLI8jhByhlO7oqnjo7sCGDRt6rVy5MptSigsWLDgZNTIpuk5ta2SSH86L5GXh+63u1b2XSkFijFUyxj4RQqznnOcj4lj1/p2glO46fvx4bVNVVfJiozEgRZxrSUyM/5WmJuv69evTogX0adOm1f3BZnMta2y0PRkX56SEgECED9xus95gCIzJzGwxVinJZAqxhgaT2+3uEGf94osvjsSjRwc+L0nytQZDAADgQCgk/biyMvWJJ56Y+Nhjj+19/fDhlFfKypLzrVafR5bpdr9frxs9+szMmTPrIs9FCCGzZ88OzJo1K6De6wGhUCgs2rZwktvtdhfzeOiQxMQG2tDgJ6GQHzjnAwlJFE5nr4aGhnSz2SwopWcopWX/SfFOkiT80c9+Vrx8+fIm/PRT3FhUZJ1GCNo5Z4cASF8haIAQkRIT49/ldpt5cnJ9VlZWt4nnUdCpjRyHAKAelTFVY2RZzos6zgPKe7yWEHI0qm5qt4Ncra9OEUJOqd+hOYSQdEJIvy5wpXdaCDabzVyYTMFal8vQz2Zr/v6r93ikoMEgW63Wjs5Aby8aAWALAGwFgCxQRPRkAJgBACmg1E2l3bAuQA/UTJoDXYMGDR2FJqBr0KBBQ89Ai3CPgCqSW1WyJRYAkgDAFgqFvksI0SOiFHFsEBQXuRcA6uA82dMIAF71XO1eF5Xo77YcEZGuVC8ojggHIaQMVHGLUtoAUbGBF8O93//+gWePHMm8x+GIm2MwBJ1C0E9kWTJlZ5+58847y9p98dDsrghHnfWOclfsY4zt+m9yV2zfvj3hwXvvvcXd2BifQil8iUg++eCD4GtvvVUzfPhwC1wYY9eo3u/SVmLsOl3kqK7gU4yxU0KIJJVUmyjL8pSMjIzqc5Sy6HeTF5FQxlpd+7777ju5bcOGwfeeOJExm1LUAeAGIWgwJcUel5FRFykQciHgrM+ni0tL61DU5datWxNJXV3SzQZDMF51SkiEwAMmk2+t12t66623hjz33HOFy5OTfV9/9VW63+Ewxw0a5Fhw442NEyZMSJRleWBUh3503HqYxAw7ycNx674mv3/siZqaYcRgqA03Awi/33O8tlZn8HqPE0JiETFXluUJAHCKMbaHUnqyI2JvVyI/P78xPz//wPULFpQ++/OfX1tXWtqn3O3WBznXjZekIJOk0FqnM7a2d++6lx9+uLir1lW/V6zh+xzVCBKZloCgfKdUI6ILAKwA0AeU2eXVEVGFXRIdqL4OZYSQsghXegYiXo2I7ghSqCPNN53+fRoxYkTT8R07XJtPn47PT01tMuv14lh9vek4pSJnzBg7KPesO99DHgDYAQpxcg0AnAaAyQAwHQBOgUIKnYCuJVZ6YgY6BY0M0qBBwzcf7YpwV3/jrviaCaBF+pYNEfsBAMiyfDUA+PF8+haiUhT5Qalh6uF8Y/Hlpm9JGJW8FRbKQXEohu9/OIraQQgpiRLIG9u7h7/uuutqlr799smn9+3LWRAI6PtKEt8cDOp3Go3+H9xzzx5KO9YvQSl1U0o3IeI2zvloNcnrfs55OWOsgFJ64j+1Z42GEALuuOOOa3Zs3nxVHCJFAPj3xx9fs+Tuu+ufeuopmVKaABfWqW3u4bvimlRX8CZE/EpNQhvHOb8jPj6+XtLr9Z5QqMX+0YdIBCGg1+sveL2tViv/zp13Fnzy6qtXlzkccaMliR+SZbZbrw8OGzKkvMrr1afabM0pSWWNjTEhiyWQnJzcoQbx7V9/nT1WCBoWzwEARut08oxAAD48cGDA0KFDt9z7i1/sWrViReYHR46kMINBzpk82bFo0SKvJEljQ6FQmBu4QCSH8+/xUxE1qoMQ4gQAm8Ptnn26vNw/1GJpXvuk3Q7VklRvNptLAGAM5/xmzrlLTaLaTyntUF3YVbDZbPJ9991Xevfdd5c+8fjj+dvXrs212+2Jv+RcmkWpyIqJ8a5yu83rGJMn3nDDAUnqOukAW44UiHSSJ0IrY8EAoJgQQhAxGQB6A4CfEFLMGNtHKa1rZYnLgso37ETEXYSQSFf6cESsugxXeqd/n4xGo+iXm1t3YN26/qa6OlNmfLzP4fPpdlRVxUJOTm12drYHuldwRgAoUR9PA8AxUIT020H5TQk3IHcl/9QTNRPRHOgaNGjoCDQBXYMGDRp6Bu3ZoIU3ileEm0J1XkbPI4+MDWx2RBBCUAgREwgEhlJKyxhjhwgh1aAQPh2KHMYLo78jHREJAKCLONwFCuFQQwg5BgBh4sHRVY7u6dOn1z/z+99//s9//GPMS8eOpUt6fWj4xInHn3jiif1ms/myxQ5KaQ2l9LMId0WeLMvjCSGH1Xj3Nt0VsiyTNWvWpDgcDsO4cePqc3JyOlW8ozIDvgXBxjlPeOFXv8pIc7ult00myGYMyjmHu+12408ffjhl7caNhZIkRcYGOnuykKGU1qsz6zZxznNnz549ft0HH8S8K8v6O/R6FwsG/WWyTD+SZTZ83LhjrZ3DZrPJb7z77uo///nPwzYUFAwQnNOcvLyyp2+77fj2VasyV5aUJI5OSvJIlOKhujrzueRk99z8/Fad2mfOnDG+8cYbQ04cPpxqNJkCM2fNKrn99tvLvV4vY4jETGmL94qJUpQAqMFg6AcA5kWLFiUsXLgwASLIB9WRelGR/GL36Oprry1fumdP1nvV1fE3JCU1cUT4rK7OVmqzuR+cOfO4TqdrRMTNnPPhQoh8zvktnPNGVQje31UEXkcxZsyYpuWrV3+yYsWKvmvWrBn4ZXFx5k632yQYEzE5Oad//fTTW1NTUzv6vQJwfmZnpEgebkyI/F5pkZZQXV3tevXVV5MObt+eQgjBYePGlTz66KPFqampAUSkQoiBalThPDWq8KBKCtV21T1pw5U+DJVZ6TWIeAwR2+PI6jRJExsbK09fuLDsq9Wr005XVMRSzmkwLs6XMWPG2bFjxzZ2xRrtRDhW/XNQZv8NA8VhcTMoxPQBUIihrkgr0BzoGjRo0NA+hB3o35oId2yZvhWLiLFwPn3LqIrkAAAglC/7gCzLAzjnDkrpEbV5sREuI0UF1ejv6H08KHsba8ShIVD2Ng0AcDRi/97hpJy2QCmFv7/xxsYXXnjBvmzLlhy/221Kzs4++6M77yy86aabKi73vISQUEQS1RDO+QTO+RLOuZ1SupMxdvBizYtHjx617Nq1Kyk+Pj5www03VEenO3UEeOEM+EQASFixYkWfPVu3xr8sSXC3Xg8IAH8PBPC5t95KzsvLK507d24BdEOd2h4QQmRJkgoRsVAIkZmUlDQub9q0Ie9t3GiabDR6EoNBD+dc/MnjMQVMJu/ChQtPt3aehx9++ESfPn08K5YvH/phVVViXO/eDQ/fdNNhg17Pd6xblxWqrLT2tVoD51wuw16fT5969dVlSUlJF7wuQghYtmxZ+pdffDHA63IZs3Nyau67776j/fv398myzMytrG0kBHSEmGRZzh88eHDC4McfvyAtQf1stSmSX6xOXmKKHwAAIABJREFUHTduXMP7AwdWvXnoUP87AZzZMTGBg253zAq325Q+a9ah3r17HwCAA0KIFM55vhBistoAf4RSuodSeqa9zfldCUmS4OXf/W7PwTvuOP7ZZ59lbNqwYehbdXWJBs5Z0GZrmDR37oFnn332UEfPG9V802JUFbT8XgmAmpZACCkTQtg//fRT4+fLl6c6a2osCf361d10yy3F8+fPrwIAEELEc87HqK70cQBwRq07j5JLJNK1F2240tMIIX3VBuRSRDwOl+aouiSKfMaMGbXrfD72ZWFhsv7MmTjZYJB1I0bUXD9//ln1u6gn66ZSAFgOAP1AiXefCEoDcgkos9KPQ+drkZ5woGtNxxo0aOgQSEecexo0aNCg4fLgcrnmg9KpebitYzjn8T6f7xGDwfC2Tqc73WMX1wmoTvF4PB8bGBbJDYhohIiubVRiA8NO8iZoOVuPCyGsnPPxiJgHSuRxIWNsh+r4jl43OjYwQb2Oi8UGRj4aSFRsYE9ACAEddU+0F6jM+R4thBgPyj0oV+9fC0fw1q1bE3/z9NOz/NXVvRjnNGQwBIdPm1b0yh/+sMNoNLZJDLThQgkTbDaImtdcXFzsvnnevMxlAHwWY4KoNt3tnNMbEOnL//d/by9cuLCyW27GZQARybPPPnvj1k8/HTFQlqVEIXC3ECEpJaXitXff/bh///4dEoPPnDlj3LRuXV/PqVPxIATo09KaJs+efa61ZoUjR45Y/uf7359nqqrqNQGUmIVCSeIjrrvuwIsvvXTq2gkTFs+pq7M+l5goS4wxoJS929hI/qLTwV9WrICsrKxIkdwOXehCeeeddzK2fPTRKL3DYQVCIJiY6Jxz552FixYtuuC1U2cm5iPicAAAQkgRpXRPZ6MyPR4Pe+edd/oXFhamxsTEhObPn39q5syZ7RY1ZVmG48ePW2JjY0NpaWkXJRwRMeYiInlk/H4TqLPfIwi28Hy95oK8oqLC+IM77phnOnMmdTZjgiPCF4iU9+9/9l8ffLAqkhgUQsSFSSFQmiDOqqTQka4ihaKeqxURcwAgQyXQ2+NKfxgU98GXnV1flmVSVlZmCgaDNC0tzRcbGxt+juMA4GoAeKGza1wCEwBgGgC8GPXfk0EhhUaB4vY6CwopdAQuf4b8j0ER47dc5t9fEvPnz79+z549yxDxD921hgYNGjR0N1wuVy8AeAmU+qBNQdjr9S4GAMlkMi3tqWvrDPB8+pYNlBFVYZHcCAAmtaZCUDi6IJx3krvhwvQtIoTI4ZxPBoBUUFJsvlb3C9jKuqaLjKgyRRzuh/MjqiLHUzXPx+5JdGfdxDnvJ4SYoO6D/ISQPZIk7SaEeMLH+P1++tOf/GTCoa1bR+r9fj1nTBhSUuqfeO65L6dPn97mPhRbJhNFJ54lwIUz4B23LVlisxQUxH5mMMgEUYAQCAAwMxTSeUePPrJm3boV3XIjLhO7du3KeOZnP1vE6uosuYhQIoR8mjHfwnvvXfvzn//8aEfOJcsyWb9+ffLpXbtSwOUygNUaSM/Pr5k9e3aNXq+/gLD+6U9/elXh6tX5o0IhXSoAFgCAp3dv+yuvvbZl6dKlo4+8++6o16xWeZDRSIFSVicE+4HdDkmLF8P//u//NovkcH4f3yXN3GVlZTGvvPBCvv/EiVQpGNQFY2ICvcaMOf3/nnyyMDy+LAxENKp1ez4o74kaSuluxlhRZ/mJgoKC+OXLlw9yuVyGYcOG1dxzzz0lNput3XVEbW2trrq62jho0CC30WhsUzBQ+Zi2RHJbxKEhUETycI1qV+tUO0SNFPjlL385atuyZZMnyTLLYYzvlmVdodHov+1//mf9/fffXxqxNuOcDxFC5AFAJgD4CCEH1AbkFuMAugKqMSRLdaUnIqJohyt9LADMBYBfdcU11NXV6WtqagwWi0VOT0/3RXw3Pg0AawFgb1escxE8CwCrQKmJwtCD0oCcCwBpoHB9B9VjLjcdoNvrwG3btiXedttti0OhUHJ3raFBg4YrC5qArkGDBg09AJfLNQ8A7oCLC+hxPp/vUYPB8I5Op+tQtHd3QnU9xKuubhu0dEQYzx+GCArR4weF7GmC82RPE7Szk1QtKPOFEONAIXXOEkIqAUDXhmDLQSmEG+BCobxLor+/SVBJtSGc84mgdAfbVUf6wcbGRlh03XW3DKurS/q5xeJOkyS+zus1/iEU0uUtWbLt+eefL4oWyCPueWzEMmEXSph8iHy4CCG4dOnS9Kd/+tM79jImZ6vR4wAA5xBhmCzrfv7LX374wAMPnOrRm3MJCCHg008/7btp06bROp0ufeLEiUmzZ88WFovlkBqP32FHsMPh0AkhoDUHBYDS+PDoI49c41q/ftg/evcOJhoMEjCmW9XUpPu9EHTWrbfCrl274NC2bTiBc7xapxOlsky2UiqSp08/9vbbb6/tbqe3w+HQFRQUJBJCcNKkSfZLkTCIaOKc56qkkA0AKlUh/XBHheCamhr992677camkpL0HETiACCVen3wmsWLv/71r39deDnPJ+y2EkIkghq1HiGSR8ZjuqFtkbxd5NZTTz2Ve3jZsqlLY2Ob+kiSAAAoC4XYHS6XddK992584oknLoiSV13pQzjnY0GZO+ePIIW6fHa7SgplIuIgUO4HYtuu9EdAEZI3dPV1RGA8KDP2ftONawAATAIluv2lNv4/A4AhoJBCA0ARz4tBIYU62vzzU1CIra2XdaXtwNy5c+fs37//PUT8c3etoUGDBg3dDZfLlQTK93K4jmgVXq/3ZkTUm83m93rs4i4B9ffUhkr6lg1bT99C9dgAnBfJw43F4Xnk7WrWQkQQQvRXhfQsAGgiytgWNyEkHs/HrUc2ALrhvMu2eR9PKW34TyUH/SehOlrHI2IuAFA1BaiAUlr/9NNP5+56//2pP9bp5OtNJl+FLLOX3W5LUa9e9uVr1nwQHx9vuohIHpn22Vozd4sZ8BPGjPnedZWVqX/S61vsL+8LBvW7Bgw4vXXHjv+a93kYZWVlMf/85z9H1NfXD8nMzOx94403mgYOHFhFKd2ljiTqUA3u9/up0+mUrFarbDKZLhCyEZHs3Lkz45kHHpj/U0LYTUlJCIzpGhGlh6urdbrp06F3Sgosf/99SHA4cJEkoQkAN3MOJQkJrj+8+eaKsWPHVnRn4pkQAnbv3h1fU1Nj7N+/v3vkyJGuix2v1u1ZQoir1H14QDUR7FFn1ncIv/3tb4d++s9/zkwJBAwpAHiUEDCkpZ179c03Vw4YMKDDc8xVkdyGEVHrESJ55BgHGdSaCVo2Jtjb23xz5swZ41033nj79/1+ww9tNjcAgECEpxsabBtTUuo/27BhWWvN/kKIRM75WEQcDUodV0Yp3ccYO9YdPJDKiQ0FgDS1pnRh6670PACYAwDPdfU1ROEZAFgNAPu6cQ2irrMSlIbg1pAESpLXKFA4vEpQaqZi6FgD8gRQXO3dVgdu3ry519133/3dYDCY2l1raNCg4cqCFuGuQYMGDT2Ddke4I2KPRrirjVTmCLIn7IiIAYXw0UUcGyKE+FBxk9uhpUje4RjwCCGrNUdEOIEtHRHTQdl4VwHAYZXk6dLYwCsFRJnzfZQxdlR1V0wUQtwghJj5xRdflDCHI+nFxERfqsGgR0mSbo6P19Xa7YZ/HzgwJxAI3BjRzRwENVINAM4KIRyI2KDX69tVCE+aNKmO6nShT0Ih6f9R2iyafsq5hIzxqVOnVnfkeTmdTslgMIiLueQ7C0opLFq0qDLsrlaF4LFCiHxZlscAQBljbGdH5nwnJCSEEFEvhEiJJB4iiDbLuRMn4K6EBEiIjTUC5yHgXM5nzF9eVmZ576WXRC5joX6cs62c0x2S5E9ISXGMnzmz6IUXXtjXE26ghISE0A033NDu14sQ4pUkaTsi7hBCDFIbYr4jhJitkkJ7KaWN7TnX888/PxZOncr4wGz2DtfpZI4Ir3o85v/78MNJs2bNOjN16tRWXQZ4fr5eIracrxc9B94Lyvu8nhByHJR4zDDpc7lu42Yc3b8/fToAhsVzAID+Oh2fhAiH9u3rBwqp0AKEEMEYO8IYOyKESFBJoVxZlscDwGmVFDraVaQQaTkr3QIAQwkh6Yg4EwA8QogyRDwKiiu9J2ICezKK8GLrcFCa3g6D0giSCwCjQSGH6kAhhQ6B8h7q7FqdhhCCgBZHqEGDhm8+/qsj3FWneLRIbgalsTgm6lgfqK5uaCV9q4PrhoWsFvUSnk/fAlDSwEYph2MDKPHGh6ME207vba4kUEobKKVrURlJlCeEGCfL8hghxMm9W7ZYb2KMLElMFMiYeShj0m8SEnRLHI6MgoKCx2fPnt08jgyixvcEg8EGvV5vZ4w1tqd5NHPw4LPrzp1LdSGCVd3bNyDCRkJw1JAhHYqvDwaDxOPxMJvNJneXgx8AoH///r7nn39+NwDsVoXgAZzz8eqef5Y653tve+d8G41GYTAYQoho45y3iLgPv89PnDjB+hICC/r1A0CUQZZDcUL49F6v+OLDD3VDjcbAGCFwvxC6V2SZJ8XGunoPGVL+0mOP7cjLy7sg1a6rQSmF8ePHt3sdtW4vYYyVqElUeajEk48HgBJVSD/Rnrrz0KFD1k/ffHP6XaGQ9DObzakjBE7JMvt+eXnf//31r6/611tvbWnt7/D8+LvEqHsePQc+bFqwA8CxKJHc1Vk+ZuPGjak6r9e0ODa2uemAEgKLzGbvWrs9bvfu3fGt1X2UUjuldD0qY9ly1LFYi4QQHrXu3NfeurM9ULmn8Kz0sCs9PCu9GhGPImI1/PfUM121BlxinXoAWA8AGwFgECj10lwAuA6UWmo/KKle7VmruyPctZpJgwYNHYImoGvQoEFDz6A9m8CwwNLlZJDqiAg7yG1qkWQBhewxgeK0i4wN9AGABwCqoaUjwo8dSC7B1mMDI8meyNhAH5yPVCuPcEY0ImKGEGIyAGQAgEQIqaCUntaE89aBiEb1nscRQqrV1yHN7/ePSDObITk5OZYDAEEUwLk8VK+X362u5m63e2N8fHyd2vHuJYRAZWWlYdXnn6dVHTyYAkJYk3JyjNfPm3c2Ozv7oqJRZmamb8o11+x+ae3aSbWhkG4ypWKXEPQNABg/deq+oUOHtotM2bBhQ69lb7010lFamgKSxLPy8kp/9MgjRZmZmd3uklGF4G2IuEMtyMerc74bVHfFAaLOIEREPbZ07oc79BMAwBJxWj+0nPXmaHA4JgmPh0qcN9+T39bVxafKMv27yeQbY7F4ZET4rcdjel2vl//vX//65FKOhv8GqKTQccbYcdUdkIeIebIsTyKEnKCU7qaUll7sc3xo+/acuxjjw3U6GQCAEQIPmM2eT5zO2JUrVw6cPHkyi2pMCIvkkfP1Iu95eA68XY2493fnPaC6/8/ed4dHcZ3dn3tntepdovcOpgsQzWBww8ZJcMHBNjFxi40bMe5xfo4dO/7yuddgOy74c4kLuCaOjTuIKkTvVRQhIbQqqKy0u/e+vz/uHWm0rISEdoUk5jyPHsBezYyuZmfv+573nBMmKgI808sBOPTPVO/3c17IOf/Oryl0uZSywtIUClpDUA/HWLPS+xmGMYiIBhLRUSmlaTEbSjR3BnpDUAJlv/4LlMpvJIDz9dcOqKbQPtR93Y051ylBCGE3g2zYsNEWYD5HT1YPSdQQO8G9AOW+laAJa6v7VqT+f+bQM0HVL24A+aghyYvQCPcty3kd5BdRRTUq8gRYorFQQ9juZ4xlaXK8UBOZYzWRPpCI1nLODzWUxDzTQJZYMKjh4W0Aurnd7l4GkTGsXTuIeOVGzYTwtRfCl+Tz+Xbt2rXroosu2qTJwxLGmJBS4r///W+HDUuXdvEWFHQIS0qKG3L22YcvueSS3JMR2fPmzVs7Z+XKoRdVVkbcxJgUAF4l4hVRUeXz5s1bV+83axQVFTmee+65wduWLesnq6qcMZ06uS6+4oots2bNaghp1SToPf8ewzD2SClThBDpeoh7ImNsqx5AzgUCErZmZJJ/FJsEUKzXeC9jrDA3Nzexqrx8UNjRo0Vc1w8bq6rClh8/HnM7gAeio487GMM2n8+41u2O7DNlyqYFCxZkhPrnDwY458Wc8++J6GchxFlSyjFCiFlCiBId6bSOMVZn/f3pp5/2SnC7I/4YH388TA9h9HE4xFVhYd5nMzMHuN3udU6n038wwaybAq25NQfevM9Dtj8PDw8XxBhVELEky38vJ2JgDBEREfXucRljPofDsRnAZillqq47R/t8volQwwhr9TBCUPbjei32Msb2khKgDGKMdSWi84ioTEoJNE890xznMT+PG7J2AsB2/RUHNXxsDiEXQCnYN0KVw3WdK6Q1k8/n46F0orBhw0bbg02g27Bhw0bzoCGb2oY2jAJ/s1KuJ/iR5FZFhNU2sBJKTVgC4DBqZ+s1KndLF8Fmzpu/IsI/5820RC5gjO1Cw20DtxDRFillHyHEBCHEb4UQLp33tzkU9lwtGScZTPC3oa6AWud9CQkJPFeI/gfdbqNbZCSH11vFy8uPrywsDGft2hUkJCRsMAyj+l4tKipyvP3880Pa797d/tK4uAqDc1q/dGmfD/bvT/r9vfeu79atW73k4z//+c9f7rnnnsp//fvfY96oqIiOjI4uP3/atMxnnnlmZUN+zqVLlyYvePTR89KKi2NvjImpOF5Z6fzPd98N//P+/SmvLFy4pDF5bk0BY0w4HI4tRLRL562lSSkvlFJeAFX8cdRWNZsZki4oZa+VsD3hPu82aFDyF8uWDZsRFcXiOScPEb6tqIi5EaAB4eFVAOBgDHdFR1d8XFYW+8knn/QZOnTo+sWLF3f+96efDiw6fDg5JjW15Nzp03fOmTNnfyjVJlZ4PB72yCOPjPjh669HlJWWxnbo0iV3zg03rLr22msP+L9WqwO+1UTwEN0Umi2EKNT27htMMltKiYKCAmdMTIzP6/E4kxwORuHhETAMBxmGgxuGI9XrdXbu3PlCn6/6FrA6Jhywrjn88vWaE+lnn733h23beq2tqnKOCg/3AMAytzt8jWHISydNym7occx7EMAW3ZgcRURpPp9vAoB9uim0MxjNAEtjkxFRoR4O6QqVMehkjPUDUGhRpQcb1Z9VIcapEPUEYK/+igIwFIpMnw31GboBqjHkbzsc8p/JVqDbsGGjjYDQ8LrpVGsmQLlvxUPVTnEAYiwkeZh5fCLyafctN4LjvuWkmtzg+uKSTEvkIgA7/ay/j9f3eW8YxldSyp+0Nflon883lqkomBWnYgvd2kFEhibJkwOsudWG2lTYFkZERGRHxscPzDp0KGZqfLzBpBTM7S47UFJScdDjiZoQG7veMIx91vMsXry4y+5FiwalE6F7bGzlwYMHEzPffz+pvLzccdVVV9VLYo8dO7bohVdffe9/n3hi6h179vQAQH0GDNj9yv/7fz8OGzbs+Ml+RiklHrznngkiM7PftU6nt6PT6V25a1eXxc89l0pEP53s/MEE57yAMfa1lDJTW5MP8vl8Q6EGTbxQNZM/YVtokuR+hG2t+3zUqFGp377zTu8vKyoiZkRHuwHg0/LyqCQidl1ERIVD7/cHORzit5zLt5YvHwgg4+DBgxGvvvrqoF2ZmT2JiPUdNWr/Lbfcsq05BrJNLFmypN1Lzz8/bv/OnT2cERGesZMnb3r88cfXJCUl1eq9aCJ4I4CNQohOUsoxUspzpJTnMMa26qz06iih8vJyw+12c7fbHRbNGHOGhTnJ4XCQYThgGI52hhEW43AYXq/3LsMwAPVsK4Gqm7L1AI5JlBefLmJx+vTpR9587rnj/ygqin80MbEkjDGUSckWlpdHxfTqdWjUqFENVpFzzo9pZ4nv9TDCKN0/KmWMrTMMYx3n/KTvq4aAiMKllDFEVEZEx6DcE/tA92M451MsqvRgoyHK8NN5nuMAlgJYBlVHjgQwFSrjfCdUzbTH77ghr5lsBboNGzYaC5tAt2HDho3mwUkLEctEb53sl27oJJCyDYyjGttAUxHBUJNHXglVqBYAKEVNs6ekIdfjd16rbWC1gtwyKR4o5y2HMbbZVEQ01TaQMQZzsl1bk0+QUv5GSjlVZ3xntSVbQrO5Z5LkFqLcnBT3zzMsZIwdYzU21OaaV5Pc559/vrHg2WevvCc7u8NtCQnUNTY26gePp/03MTFixuzZuwzDqEUo/fzzz+2i9+5Nvb5Hj2NxTqcEgBHJyeUv79vX7scff+zw+9//Pru+n8HhcNDzzz+/yvPkk6vz8/PD27VrV+V0OhtcEH38wQcDzyoujn28U6djZkMk3eNx37pvX6fFixd3vf766/c39FgNRQAluXXN/ZXkXigSzQCQpxXBmznnjWrG3DFv3rp7d+zoemVBQdIkzsURIXghEUt2Oj2RFpVyuDoZud1uxz//+c9eXy5YcM6Yysrw4U6nZ/exY4lf7N7dNTc3d+Wf/vSnE2zBQ4Frf/e78zcsXTpqBhEGck7f7tzZ+68PPthj//79/01MSKgszMmJi0lJqZhy7rk5aWlpJQDAGPM6HI51RLROStlVN4XOl1KeCyD7m2++qfziww87lBw5EhMeG+tI6d7d8e3evfhtTAycjBETwrehvFzsIfJM6949yzCMnZxzF4Dy00WS14e5c+fuylq1qvvcTZv6Disri5IANhmG6DxmzLbrrrtu30kPEACc8wLO+Td+TaErhRBlFlV6nbmxQK3ni2lxb+bBm3aN5jNdosaucQ9TcDDGztKq9HzdFPLPSm8KWovlYQWAVfqrM1RTaByAyVAE+3oodboIwrlOCiklh90MsmHDRutHUCzcSblvxVJtq/VY1OSRh8R9S5870jLkmki1CVvrwGUV9DAxgMN+JHlZU9SenPMyrWbN0NbkY30+30jG2DbDMJabauC2AlLq/boGE+JxYlZzIYDtAQYTCADCwsKQdvbZhz9+881zY/bu5RempITlMpbwuhDxKYMGFV911VW1yLySkhLHth9/7DHN4RBTO3cuAYDhyckVcUeOxH390089Ci+8MM+fKPXHtGnTjk6bNu1fBQUFYZxznOz1Vvz444+pRRs39nwsPr5sVHR0JQBMiY2t+N+8vJT/fPzxkN/+9reHgj1gW4eS3Gr9bZLkBFWrGlBDIpWMsU2c80zO+dHGELZTp049tnjy5E2P//jjiB9drvAOnMt/ud2RXRiTKZpQN5HAGHm93rDc3Nzw+bfccmF8dnbnKxwOLwD88MUX7e7ZsKHb82+88d8uXbqE1I0KAL755pv2f7zlltm9KivD5zFGBeXl0f/67LPJMzZu7H7rHXcs271lSzIRsQHDhh2bPn16bnR0tAAAwzCOGIbxOREtEUKM0Hv+oUKIgvz8/JzXFiyI3756dTsuZVh05868MCrKyAgLi5kYEwNIKYTP5/uqtJTajxiRGx0d/YulJ9Pi9ovx8fG+a2+//Ze3n3nm/A2FhQn9AbYJoOKkpJIH779/2ancv7ru3ABgg5Syg47FGufz+SYxxnbriIE9J3vW6udLLcczs3ZC7Wd6OdTzZTeAUsZYOOc8xVSlk8pK34HGZYLX+yOalxik44XqPARgv/6KhBpAHgHgaqjPW3MAuRjNY+HOW+J7wIYNGy0XNoFuw4YNG82DBlu46ynWjqidR24qIpyorYioJKIKqAaPlSRv9EQtBbYNtDYerLaBRbr4Mm0DzcZDcXNsRg3DOGwYxkdaiTlBSnmelHISY2yNw+FYXZ+9WUsCEYFq1Pu11OQ4Ub1fClWQ5THGtlkmxRs8mBAdHS2eevnlfz/6pz+dc9fevV15WZlwJibKy268UV533XVjvF7vAG1NnsUYq8o5dCimNwCTPAeAcMOgAeHhnjW7dycCyG7IeZ1OJ51KcyJ/z54O0yMjqxwWYrSr0+nrT8R37dqVBFWENRoW9Y9ViVIXSW422UwleaEmbN2MMRCRQyuq04noIp/PN0qv4SbGWIMaXyNHjix5eeHCT994441B32/Z0tkZHV2VfOBA++/y81N+RwTThu87j8eZbxhizpgxRz5YsGDCrzyesLvbtXNx/f/fLiyMe+uLL0YevvbaPaFuBmVkZCStzcgY+RRj8gan0wcAdxJhRlWV8z9vvnnBlX36HBgZFuY74PUa769c2ePw73+/4Ve/+lWFtfmAGsIWABwZGRl9Pnz+eYwvL6cJMTHeQ/n5VQtLSjzLKyv57L17xTSHw1MgJf+ciMcNHLj34osv/sEwjBZtvxYdHS3efu+9Je++++6uVatWdeWc0zXjxx+65pprshszTBIIVoWKlLK9bgql+3y+s82mEGPskOVet5Lkyaj9fGmUEoVUBMhAxlh3IpoClZWeTcFRpTcXgR5MhUOO/voWwCAoMv0KKEJmoz5XSO9VW4Fuw4aNNoJGWbjrzzj/PPIoIoqwHoOIqqCeycVQOazWPPLGum+BiGIC7N9Nu/UIy8tNJyhrjExRcznkMMYqHQ5HBhGtEkIMl1KO9/l8f4CyNF7OOd/fEgcQA4GIwgIMuQZS73tRs3+3ZsAXskZkNd93331by8vLnW/++98j38zPjyKHw9131Cjfk3/9a7zT6bzd6/Vu45yvMAzjSHZ2dlRYUVHUkHbtatXgw5KTy7/LzY3fv39/VFJSUr3DjSZSUlIadT8CwJYtW5KSPJ6wkcnJ1efgjGFCTEz5j3l5SUePHg3v2LFjo/dnFpK8VjxVHSS5OTS/z7KPrLWX1CRmOhEN1fXTRsMwVnPOjzXkejjneOUf/8h47bXX8n75/vu+m8vKIjtFRlYdWrdu4AYpWbpSWKOSCF/4fI7uAwbs+r//+78+4dnZnV5KSirqGBYmAOAyn4/fcPBgh3feeafvQw89tLmx69JYPP/ssxP7VVaGL3E6PVH6/Tbd5+Mzdu/u9dMzz4RNSUgoYwCVFbPcAAAgAElEQVRtWrOm15bMzIP3/eUvWyIjI02xQrJl7SMAoLKyMuXJv/41ha1fj7nR0SKWqOq/GzYgy+1md5WXYwbn3s6GIb7zesM2Rka675s16z+GYQRz2DUkmDNnTvagQYM++fDDD/vtd7miB3TuXDJnzpyd/fr1q8vuu8HgnOdxzv9DRN/pe2+UEOJqbZG/jnO+AYDDrJks/YFkqF6YiSpY4tj03+uMBiMixmqy0ocQ0SDLAHJTB5mai0A3+4DBOI8bwGr91QmqZkoHMAmqt+OGHXtlw4aNFgabQLdhw4aN5kH1JlCrGGLJYrUOIFZKGQUAQogLABzjnAutiKiEar4cQ22r9YpTUERY1bX+iohAtoGFAHYEmM5vEaSVVmJ+oW0KxxHROK/XO15bc63knDfY6itU8Gs8BCLJ61Pvm2te1FAy9mQYOnRo6eJ///urdevWxR87diw8LS2tKCUlxSulbCeEGCelPFdKOZkxti4mLu5IkVLw1EKR1+twxsWFwrq5FsJjYtxHXS5rwQofEfIBdIiJqXdowNJk87drTEYjSfL6oEnM9US0XkrZQ0qZLqW8REp5rr4P1zTEIm7gwIFlzzzzzBrz34sXL+78Pw8+eNnM0tLYCx0OX46U/HOAdR85cluHDh3cVFgYNz02toxbru/XcXFl7xUVJWRkZKSGOu/w22+/7R4uBP+d01n9e2AAugGIrKyMnNuvX3hySkoshYVFfn3wYOymrKy0silTeEREBGBRNTPGdjHGXERU+Oozz4wef+xY5/t693awiIhIJCUZA/Lzy27JzxdHunbNeT4vL8UZEVE18uyztz/00EPrIyIiTstzyOfzsc8//7zT0aNHo4YPH+6aMGFCvXaoERER8qabbtp30003nZLi/GQg5UwCznm2lLKciHoTUXchRF+/l5pOFXmMsa16/V36+dKoOAStjMukmqz0/oZhBEuV3loU6IHggVJRbACQgprMvzAAY/X/34oQWN/bBLoNGzbaCEwFOoB63be6E1F8VVXVlZzzcv09Zh75MTTdfYuRct9KDLB/T4R6rps4DvX5epQxtt1v/x7y/XJDoPera4lonRBikJRyghDiWiHEEcMwMjjnO5qieA8WiCjcsua1lM2ovX+3RvccCqDeb/K1cM7x2GOPrS+YN29LVlZWYmpqatXIkSNLiChMDyOME0LcJIQ4GB8fnyXCwkRxVZUjNSKiek9VWFXlkE6niImJCWnsVHx8vKecMSoRgic6HNX3ep7X60BkZFVcXFyd56+HJDfXvVEkeX3QJOYXpFyU0qSUo30+XxpUHNFqzvnuk92HnHPMnTt3z9y5c/cAQGVlJb/8178O/8OOHb0uq6qiFM7lv30+x+6YmPK/3377mvfeemvYWIBM8hwAUh0OOZ4xWrVpUycAISfQ9+/c2fNhzmWU5b4M5xzDAHYR53GzRo+WFBYWlVdVFbXw6NEBGzduvHj06NHmS01V8zHG2E7GmGvRokUxh5YvH/t/nTuzDgkJceA8alJiYnnprl1idUJC3hdERlV5eWTHwYP3/vnGG7OmT58eCvvwBmHbtm0xv/zyS4fo6Gjv5ZdffthU19eF0aNHF48ePXpNfa85Veh7PULfs1lE1A1ANynlFCnlFMtLzV6YC8AWs2Y6FdczVjsrPRZqALkbEZ1LTVeltxYFel04or/MAeQR+k8CcBGAdQCOBvmcJoHeIvqZNmzYaB2wCXQbNmzYaAa88847nSorK9Ouvfbafg6HIxI1m1BmUUQcNwxjsxCir5SyO2Nsi2EYv+iNeoOgCXWrbWA1WY76bQPNxoNZFJe2FjUCAHDOj+t85aVCiNHapnC0XsMMznl+KM9vabLVRZJbGw/FUOt+gDG23q/x0CyZ3oBSPVv/zTnP18MIPwohxhDRqMmTJ6f/d9u2gqVFRY4J8fFHGGNYV1AQvT0sTKSPGtUgpUBTkHbOOXt+WLiw44jS0sgJMTHuKiL2fwUFCUdiYiquv/jig35KFGuTzZ8kr54Uh8rHdmmSPGjqH6YiBrINw8iWUiaYa+jz+cYzxrZzzldxzg819FyXX355TkRExCdvvf562vN793aNjI6umDh16tY///nPG3bs2BFDnFOpsmyuxnEpOTEmIyMjQ3Yfmaqr1NTUOMk5L3E4nCkOByfGWB7A3R4PehkGnO3b95Lh4ZXM662YmJxckLFjB7Zt27ZuzJgx+1mATMOcnJzwkpycqWeHhRWHuVyVFBbmpOjo+OHt2sX1q6zk42fPdl9zzTXfnW6l1PLly5Mevvfei9x5eamGEMa7Tqe35/DhO//x+us/JCYmhnLdTZVdIFWEdfipEupe3wmgSr+mKwCDMXaYc57FOd8brCa5/l1kM8ayqUaV3g3AVCIql1Lu102hxjgitGYC3YoCAN8B+BHAn6AaY5cAmAZFoq+DUkEGBZpAt5tBNmzYaO2gv/3tbyNnzpwZ3atXL1Bt9y3BVB55Bed8KxH19fl8HaCii5YahrG5MUO+VJONXatespDkVtWddf++wY8kb7b9e1PBGJMOh2MLEW2RUvYWQkzUMTAuzvly7aAU0mEsIooIUKsGsriva8i1WdT7JlJSUrwXXnhhdS3JlC10JhGtlVL2F0KM79Sp06V9zj+/bNmPP7oSPZ5dKU6np7Cqyvj+6NF4x5AhOX379m2ygrY+TJ8+Peff775b8o9jx5Jub9euMN4w5JaKCucXbndE3ylTtkZFRUkppbVWTUY9JDnUuptOc0HPx2aMlTscjqVEtFwPdIwVQlwlhCjUGd/rWQMd1iIiIuS7H3741d/+9rcRi5cuHeRxu8O7DRx44Knbblt7zjnnFHz43nu+0gAD4aVEzAgLC8qAel0gbf0dGRmJYiEc5HQaxDkHY2xXZSXryRg6d+iQJOPiYuH1ujsAFb3Ly92rly7NGTt2bBbn3BVoCGfjxo1j+/t8oovbXYTKygIZFRXLoqMTzmvfPvxYfHzEm4sW/WQYxgbGWLNlvPvD5/OxO++4Y0LWjz8Od1ZVhUvG5D+ff77kzgcfXDJz5szDoTqv7odF1RFTZRUvmAMh+US0C8pavAuABAClnPNNeg2D9t7VPbY11gFkKFX6YCLKo8ar0ls7gW7CC+XYtRHAdABDAJwFYAyAXKiaaTOCNIAshAi5M5gNGzbaFmwC3YYNGzaaAV999VX+nj17Ep977rmOt956a+511123Njw8/ChUM6a6SWEYBjjnYUKIkVLKCT6fb4gmgZeZJDDV2H4HUkT4Z2ObE8tFrMY20Gw8nFRd29rAGHPrYnylzgkb7/P55jJlZ5zBOT94qj+zJrBqWdwjcJNNAijWjQarxb1LE4ctWiHIOS/lnP9AREu7du06YthvfjNhw88/j9iSm9vHU1KSm8fY0c7nnbd7ypQpISfQb7nllt0P7tuX8r9r1w5KLSxM8RiGUdm5M66/+Wb34MGDb/R6vc1KkjcUnPNizvkSIvpZCDFMSpkuhLheCJHLOV9lGMbWhtwH06dPz5s+ffp//P/7iBEjSqK6dct7b9eu7gPDwz1xhkFuKdlbRUXx1KGD6/zzz2/SpLZf48HaZDMbD85Zs2bhrQUL8GhpqfG0YYhwInnU6/VtkdJxRXz88YTc3A1MH4h5PEZFSUlCZWXlQc55UaBzJiQk+JjT6c1xu8MBgHm9HlZcfKxIiMLCysp2SUlJMVopVcA5zzQMY2Nzq7oqKir4n+fPn947L6/9w9HR5f0cDt/3lZXhj2VmDr7//vsrXn/99WVNOT7VzpJM9lv3RNTO73RpVcQmP1XECfc6EYVbrAqvEUIUc86zdHMymE2halU6gB4A+hmGMYhqVOk7iaghDbPmJNCbo3livtfXANgFpUg3lekFUJl/G6E+r08ZtgLdhg0bbQHXXnttpcvl8r744ov9LrzwwqL58+fvGDRo0A4oUq/afYtzDs75D0KIzlLKSVLKGVLKyZzzDL1HEEC1I1GgbOxE1M7GFqhR1+72I8iDRhy2FOjBz72GYezVazhRSvlrKeUUzvlKHed0Sjm9ln1kXSR5pOXlgSzuC7Ud8mkj/xoCxhgZhrHDMIwdQogu55x//sQfKiv7fbB7d2dnUVF+XllZbmXv3kevnjNnd7Dzx/3Rrl07z5w//jHj/ZdemrKhuLhDPOeO4shIo9NFF3nm33//WK/XexFOHOh2hZIkbwgYY8LhcGwmos1Syi5SyrFSygullFMZY+u1Kj1g7WBFUlKSVzt5naBcHj9p0oEv167tv6y8POJsnQ+/orw8YjXn8sJJkw409WfQgzjWaLDqqCroIdff/Pa3eO+tt3CplPIsxqTw+WhJZSUXRKyP07nBuXdv9WB7VVFR6vGSkpz6LNdjY2OrDgPMS4QwgHh5+XGUlR0/WFKSEt65M9PxdlMZY5sNw8jknDfVKrzR+J//+Z8hm7/5ZvSDhiEuj40tPSYlf9LlSnz20Uenjxgx4t0+ffo0KXKPlFtFkpSymiC3DBdb+2HHoe71g1q8YI3Bk37HhJSyq5RylFalT9WD8Gs559nB6ifoQeb9jLH9VFuVPpWIKqhGlX6yWre5CfTmeDaY7psvA+gLZfF+MYALoQaQ1wNo0vvW5/PZNZMNGzYaBbO3acOGDRs2QoxNmzZFPPHEE3euWLHizsrKyoRrr7122x133PFLTEzMCU1zImJSyiQp5SgiGgbVaCiFms6MRWDbQGujx/z7KTU+2gqIiAshBkspJwBoB+CQVqQHtIfzU6L42wYmIECTDbXXvjCQurY1g4jYkSNHhh44cGCcYRjtO3XqVNixY8cVujkZFMVNPUryJCKK3bNnD3bv3o2wsDDviBEjXKmpqQXQtmqniyRvDPT7ubcQYiyA3gDKNQm89lQJzIyMjKQnH3rogthjxxL7QwWGHUtIOH7jfff9NGPGjAZZZ2sFkDWP3EqSW/M7S1GT9Vb957PPPtv+1eefvzhVCEcfAFlSgjmd3ju6d8+5a8CAw4b+fXx46FDyDykpxx955ZVf4uPj67xn/jhv3ljXN9+MeiQ2tmx4RISnWEr+dGFh/E8JCccXfv75R6mpqZ2klKOJaCAAL2Nso24KhXyYAwDeeeedHq8/8siln0dFufs6HNVF9+vl5VHPhYdX/WfZsjeTkpLqVbLofXd0AFWEf6RDtc09NFluWf8G53f6n1s3J9OIaDAAzhjbYWkKBb0oIKJo/fvqASCCGqZKnwpgKIDng309fjgfQH+oBk2o8TCA/wDI0v9mUGsyEsBA/e9dUAqLvTiFBtW4ceOuOHDgwCNEtCgYF2zDhg0bpwsul4u98MILY5YsWfL4xo0bJ0+cODH3rrvuWpaenr4n0OuJKEI7eKUD6AxVL5UCcOJE229T0Vzkt38/pc/WtgQpZYoQYgIRDQXgYYxlOhyO1YH2qpb9jP9gsbmPtBJYZoSMufYmQV7EAmQGt2ZUVFQk79ix4+zKysqz4uLiWM+ePTdFRkauDNZelU6MBqulJC8rKzPWrl2LsrIy6tq1a+ngwYPzHQ5Hod8evkUPhEgp44QQo4koDUAkY2yXdvI6JRcqj8fD7rrzzgk5K1cO7OfxOABgV1iYr2N6+o4XXnklw+l0nvR9bxmkD0SSWwdxvKix/jbX3HXkyJHymTNmzCg6cqTDCAD5APYBNCE5ufD5oUP39oyO9gDA4YqKsKfz8xMG33DDmvriuDIzMxMe+sMfLr+qqsr5h8TE4xGM0Y/l5ZH/U1kZPmbOnF/uv//+bC3IGAVF4h/Wyv5tzTXMf+748bOnHTnS7m/x8aXmfyuRkp1XWho7fs6cHx599NENJzsGaQW/Zc2tJLnVrcIUjbj8vk45Bo+IIvUg/Ch9PpceQA6Jsp+US4KpSk/SopmTqdLDATwA4BMA24J9TRZEA7gHwL+g6pVQ4jyo2ugly3+LBTAMqm5KBFAIRaRvgPp8aRTefvvtXo899tgQt9s9oumXa8OGjTMBNoFuw4YNG82M9evXhz/99NNzly9ffldxcXG7c845J6dHjx7iyJEj8bm5ubFz5871TZkyJQa1bQMroJpAYVD56Os553tYK7MNPF3QxFFfIcREqJjmIqYsjksBJFrIK2sB7EMNgeVPkh8/E5tsQoguUsrxRDQAysEg0+FwZDaEBPYjyWtl7EEVRSbMaAHrurtaOkneUOjmZLoejOFMOUysPhVlwMGDByM++uijXocPH45LTU0tnzlz5r7+/fvX+l0QkZOIkqWUgUjyKMtLy1GbJLfmGtbZeMjIyEhauHDhkMLCwug+ffrkT5gwIeenDz4Y0vXYsfi+YWG+A16vsT8+vmLy73+//je/+U29P+PRo0edd91667klO3Z0TxHCKAVQkZBw/KZ77/3x8ssvz7GsYZwQIk031qIBZBuGkSml3PHGG2/0+v777/sCYJMmTdp788037w5WVvoTTzwx+IfXXrswMz7+uDV7fkVVVdj1QjheW7Ro4bBhw44DtexJ/a0Dk6Ge5SZKUEOQW9c9pM1NTTaYTaEUqGa22RRqkiKkjvMBKq+2P4AU3RQ6Wocq/VwAgwG8EOzr8MMFUMqGV0J8HgD4C4CvoAhyf0RCWRWOBNAe6nNpA1Rj6KSqKxPp6elXHjp06CEi+qzpl2vDhg0bLQOPPfZY2tdff/3Y+vXrzxswYEDR5MmTXWVlZVE5OTnxsbGxYS+++CJHbUVzJVTtFAlFmO/knK83DCMfjcyuPVOh91ljiWgUVF20hzF2GGq40jpcbN3PlOLEeumMHejW+6xRUsoxAGKZckNb2RASWJPksX5DxWbdlIjaFtRmtIB13+5q6SR5Q0BEDiHEUD0Y0w5APud8tY4ZaFT/Q0qJzz77rPOKFSs6A8C4ceNyZsyYccThcFTX81QTyVYrKkmvewJqejMCNbWq/4BxnTF4JSUljpdffnngunXrukRFRXl+9atf7di2fn17T1ZWjyFSMs4YbQYYHz784P2PPLLuZFnhr7zySt8v33prYlxZWVQ4wFxOp6drevrOF19+eZlZ+xARl1L2E0KMAdATqpbOMgwja+PGjbRgwYJheXl5CV26dCm85ZZbNg0dOrS0vnM2BukDB976sM9nXBMVVWtI5vKSkljnRRetWbBgQYZ5jZbhBP+YqnjLt1a7zVkHjPUwTsgGcXQfqbsWtgwCQIyxrXoAucHxbI08Z6weQO5OSmlfRkTmALJVlR4B4H4AHwPYHvQLqUEMgLsBfABgdwjPA6j6rB/qHnDuDlUzDYJy1TAHkPeggQPIb7zxRu8nnnhioNvtHtX0y7Vhw8aZAJtAt2HDho1mAmMsHcD1UArUPlBELgMAp9OJzp07U9euXd3XXXdd9kUXXZTtr2gmpaY+S0p5NoBUAPsNw/jFMIwmW4+1NQQga63Nnji/l5cCyPFrPNhKlHogpUzUjbURUCTwRsMwVjLGSmySvGHQjbWRurEWD+CgJtJ3NLbhRYEV/IGm893wazzopkMhC6Id+vbt22O+W7KkU2FOTlxMSkrFlHPPzUlLSys5+XeqBteXX37ZaevWrckJCQmVV1xxxYH27dsHbLwSkSGEGCilHO3z+brNnzfP+8PXXxsDpZQGEW0C0GvQoN2ff/XVpydrQjUEixcv7vzMfffNfMfp9Ix2Or3a/9TxTGlp9EcpKb5PlyzZ73Q6TTVKoOGEQKqI0zoApZtC3SxNITDGtumm0ClHXpzknP6q9AqtSt8ORXycB9UUeTHoJ6+NaQB6AfhHiM/DoBToX0AR4/WhI1RTaAiUqmQ/FJG+HWqoq06MHj36tzk5OfcT0ZdNvmIbNmzYOI1g6sPnb1BN9D5QtVO1kjw1NRVdunTxDhw4sPjZZ5/djACKZillkhDibK2mrtS25JnB3O+0BdRB1lpt7q2uZ14A+YyxPL+a6ZRVnm0deq86WEo5DmpILk8T6VsBRDeRJG/x0WDBgN6r9pBSjtWDmG5NAmdyzo838lj+97uVJE9Ejc29vxOUdf2DNkhfUVHBv/76647bN2xIJSI2YNiwY9OnT89taN2yd+/eqC+++KJbZWWlIz09/eiUKVOO1RUXoIe4RxPR8BUrVoTdduONMry8HEOIaCOAivDwysefeuqjK6+8Mij55BdNmXJl2v79XV+Oj1e/I86NI0Rhvyori5790EMHZ8+eTVQTU2UdTjBjqqqHunVM1WkfgCKiaCHEcCllGtR1H+Ocr9VDHUEn8S2q9H5Q9ynpAeQdRHQEalDsPgAfAdgR7PNbEAfgLgDvQxHVocSFUJ/7JxtwjoCql0ZA1U9lqBlALqzvG1999dW+Tz75ZB+3253e9Mu1YcPGmQCbQLdhw4aNZgJj7NcAHoWyZ92jv/Z27949e+rUqednZGTcm5eX13XWrFk77rrrrp/btWtXHOg4pOygBwghJgHoAEW8LeWc7z3dRUVzQk/j+mcamspaf7tGMxu7utmjC+iRRDQEqrG22jCMNaGw5GpLsJK1Usp2uqBrj5qGg4lAJHlhXTnNZyq0MqC/ECIdaqK6RNu7r7Pei6Qs7Mz73Z8k9x9OqFYy678Xcs5doZzOP9147rnnJix4+umpb4WF8UucTjAhxI9VVeJKn4//9sYbv3788cfXn8pxtSoikYiSfT5f0g3XXjvat2FD/K0xMaxXRARfWlaGd4TAFfPm+ebMmXPUbzjBbLS1imY9EUVZmkJJAAp0U2hjiJpCQABVupQyDkAX1LbuCwUugiLxF4T4PBzA/wPwOVTWeUMQBmVfOBLquVAJYBNUUygv0DekpaVdlZubO5+I/tPkK7Zhw4aN0wzG2DKoPc0eWGqnefPmObOysh5ct27dJUOGDCmcN2/eygsuuGAL6siAlVImaFvyEQC8nPNVhmGsbst7In9YlLX+duvmkKt/NnYtBTmA40TUQ0o5FkA8Y2wP5zyDc37A3s/XDStZq12JekLtO6L9X4rAJLnpSNTmSfKGQg9xj9HvZ6ce+lxtVQLr/WSMVdGMwHFJhJpINn+SvE1FsllRUVERMSk9/fa0wsKot6KiWAxAx71e79WVlXxLcrIrc+PG16zK/IZC7+ujzJiqzz//vM8rjz7a73IhjEvi45lLSvbP48fh6tULr737bmFsbKz/cIIrmMMJoYTux/XUA8gDAAjG2GbOeRbnPCdEA8gxeti5GxFFACgXQhyGygf/EMDOoJ+0BvEA/gjgPajP41BiGpRbQmPqsw6oGUCOgMpIXwc1gHzCcNcrr7zS75lnnunhdrvHN/1ybdiwcSbAJtBt2LBho4XA5XI5HnzwwWuWLl36QE5OTq+ZM2fumj9//k8dO3YMOEGpp7H7aSK9M4AcTaTvaivNDKqxQa5u9FiKX2vzoRI1mYbVjYeGKJqllPFCiHGk7KCJMbbOMIyVnPMGKWbbIjRJnmhVRViGE05QkkPZDBtQ079xAI5yzpcbhrG1rTYfQgEhRCcp5URNKAKAC+rejkNtCzszw9M6nX9GDyecf+65Mztv3drvS6fTS4bhgMPhIMbYteXltGfQoIL/LFnyBqvDQpRq8iSt2XrmPZ8IS6yDy+Uq/PvjjzvWL1sWBY+HWGRkxbgLL1z/2GOPrQgLC2sTm2rdFOphaQpJi1Xh4WDdX1Q7D74TEfUBkEpE0QB8jLFvLar0UOBiAF0BvBai45twAHgIwGdQJHhjkQSlrhgONRyWC9UU2gLL2owcOfLqvLy8O4nomyZfsQ0bNmy0cDz77LN9v/rqq79mZmbOGDBgQPGdd965evr06ZtQh4WrtiUfr/f7kjG2xuFwrGQhiC05HaAaG+RAJLlV4SkBFPsPFjeErKUaN7SJUJbahw3DyNC1Z5vYAzUWFpK8lvOWZQ95gpIcyhEqEapuErr2XMU5Dzg8b+NESCmdUsoxUsrRUHVSGVQskoET4wX8hxPMP4vOxDr1/fff7/bwPffM/skwfMMcDg7DcJBhGKt8PvzK45EvvfHGp9OmTdta1/drEUOSlLI6osrSJ4iwvPT4okWLKt999dXoimPHuABEaq9eRx54+OFv09PT61UItyZIKWOklCP0AHI8lMvEWsMwNtdVe54K9DC9ue59iKgrEcUCiGCMbQGwUqvSQ4EEAPMAvAtgX4jOYeJiKKfOV0/hex2oGUDuAdWr2gxVN1VHyb344osDnn/++c5ut/vsJl+tDRs2zgjYBLoNGzZstDC4XC7jz3/+85VLly59KDs7u+/ll1++Z/78+T937dr1WKDXayK9lxBiMtRm86gm0re39GaG/gyK1ARKIJLcmmlYAUujx/y7tmxssmqciKJ8Pt8YIkqHmmjfbBjGcs55wHVv7aiDJDftp60kuVXB7z+cUMtKTd+LvYUQ46DsNo9rZX9Wa1Hhhhq60ZZgabQlW9Y9ATVkrdnQ4VCNzp2Mse1a1Vx2JpLk9eGciROvHrF7d893wsOrGxXEuXGHx+PcPGIE++izz6oYY1sYY9kAwqi2ZaNVjVKXZaOLWWIddu3aFX348OHIoUOHlqSkpLRZ21LdFDJV6QlQwzFmU6hB72kicmr1lX+jLRnKptxEMdSaexhjxDkPI4VjpKwKg2IpacF0qOGz14N8XH+EAfgTgE+hmjinCg6V2T4CytZYANh2+PDhrZ06ddo1cuTI2fn5+XOJ6LsmX7ENGzZstBIsWLCgx6JFi/66Zs2aK/r06VN62223rbn00kvXow4inYiifT7feCIaDQCMsbWGYazgnJc164WfAojI8CPJrfsY6x5SoEZZ60+SN1lZq/f7fYUQE6FqzwLOeYZhGFvaolq6HpI8kKLZquC3qslPGE6QUsZoNfUoKBJsG+d8hWEYoSLBWh2IKFIP0/sryf33kD6o34MXwEHO+SbO+RFNkre5e7IpeP3113s/+fDDs9Y7HN7unJu9IraVMcc5QoS98a9/IT09/SDnfAfU/Zzkt3e3ihgqUHdMlRcAysvLjYlJ01AAACAASURBVMzMzMT4+HjPiBEjGmW535qgB5D7SCnTSDn0eXU/aS3nPKB7VKBjkHILSabag90pCJwHXwLA4JwLqKGRckssVjB7L4kA7gTwDoDsIB43EKZDOZE1dcA5ETUDyLEA8jwez4b8/Pwdn3zySaeXXnqpvdvtPqeJ57Bhw8YZAptAt2HDho0WCpfLxf/yl7/M+OWXXx7eu3fvoBkzZuydP3/+zz179jxa1/cIIbprRXovqEymZadbBWxRGdZlG2idVC6DH0nun2nYDNfr1NnU4wDEMcZ26qZQsMmbkMNCklttv09GklsVzSeQ5A2FlLKdVvYPhVKTmuqKNq/s91M0V6v3LWoUa86bSdb6KyNKAXAhxCBtmdkJ6v2w2jCMDcGcaG8LmDdv3tgfPvro3JUOh6+bYQCc81wiNsHrdVw6d27FfffdF4HaMQPlAI5aGj3muhefiWqUk0E3hXrrplB/qPe02RTKJYvVveUrBepZY43UsDbaCvwabT6/c0aRykrvCSCciNy6KbQDSr3VVFwCpf76ZxCOVR+cAB4EsAhAnYqeRiIGqiE04rLLLkvKzc2V3bt3r1y6dOkcIloUpHPYsGHDRqvBa6+91uWTTz55NDMzc1bXrl0rbrvttswrrrgiSxMLJ4CIIn0+31g9OOtgKlN5eWMzlYMNqh3Z4z9YHA+LKw5OtJ82v5rNBlkI0VW7J/WDGpxdqWOIWtU+1Y8kt+7dT0aSW/fvp2S3TkRhOkJnHFSdcFAPdZwRyn4/RXMtshy1h+nNPoH/gGsRY8wrpUwVQqQT0TAAjDG2RdeeDSIvzxQcPnw4Ysr48Xfc5vGEPRoe7gNjXAL8AY/H+Cg+nn5asaIqOjo6yvItAkABY+yYZcDYpXs0Z0wURmMgpYyTUo6UUo6E6rnkcM6zDMPYAjXkESWlTLEOJlCNit+sVwVqHOeqhxMC5cHrfltPqp2Vnk9E2yk4qvQkAHcAWAhljx5KBLs+41DCjpHfffddvz/84Q983LhxFQcOHNi/f//+oURk1/02bNg4KWwC3YYNGzZaOFwuF3/sscem//zzz3/ZuXPn0EsuuWT/3Xff/Uvfvn3r3AwLIbpIKScRUV8o0m2ZYRibQkUMkcoaszYd/Elyq41aKU5UQ5hfLabZQkSGEGKIlHIC1NTvAd1Y292S1L+60VZLiWJZ9zjLS4NOkjcEUspYi7oinCkr6JWtXV1huefrIsmtiuZiP5Lcmq930o2YVvp0kVKOJZV95mGMrTcMYw3nvChkP2QLhR4Msd7zyS6XK+XqK67o5D5wgF9JBIMxfEQE6tDB++Gnn+7o1KnTUaiogUQp5UAo5XGJVlOvY23EwjWUMO95KWU3KeVQKGs8J1SDh6Omqe+FbvTAr+FzKo02Xat008R9qm4KBUOV/isA7QG80YRjNAQRAO4H8AmAbcE++BdffDHmyy+/HPv9998ner1eCeA/AN4E8DURtVl3BBs2bNgIhLfffrvDhx9++MiaNWuuad++vefWW29de9VVV62ph0iPEEKM0cOK4YyxDdqWPGT7K799jD9Jbt27e1ETUeVfM5U2ZA/ZXNDk5QQiGgKginO+xjCMNS1pf9VIkrwEtQcTmkSSN/T6pJT9hRDjoSJmCnXNtNFU87ZWWO75ZOufCKxorq6XoNbAXPsG9QmIKFIPw4+Bej8d1ET6zjNtSNZyz9dyPXvllVc6L3j22eipUiKdc6yQEj8zRnfef3/ezTffvE+vvSSinrr2NBhjOzjnazjnB1pSL6SlgojCpJQpUsrBpCKxkqCeLRK1h7rNZ43/V/GpPONJZaUPBNCDiMIRHFV6MoDbAbwN4OApHqOh+DVUREjQ67N9+/alfvDBB+d9/fXXfbKzszmA/QDeArCwiTWlDRs22jhsAt2GDRs2WglcLhd74oknLvzpp5/+sn379rRp06YduPvuu38ZOHBgnZs9KWVHIcQkvWkv1krqDac4HV9f0yERyqbWRAkCk+RFra0BoJsZ/YQQZ0ORbs2e7+1HkvtbNtZHkheaE+IIIUneEGh1xQjdoEyEGkhY2ZLVFVQ7o9mfJE9CzT3v32izkuRBVTTrDM/RpDI8I5lySFjNOd/flpoZVFvR7G8daL3nTQs7V15e3vG///3vKSt/+qmDz+eTZ6Wl7XjooYdWDh48uNT/+PrZOIaIBkOpVLZyzjODmfHdWkFEEfqe91dFJKP2PV8ElcEdBWUd69XruMowjDqdUpp4bVZVegQRVUgps3VTqLGq9N9ADUe9Gezr9EMkgPsAfAxge6hOMmjQoOuLi4s/BnAOgDQAeQCGElGbjCGxYcOGjfrwySefpL799tv/b/Xq1XOSkpLEzTffvG7OnDmrOOe+QK8n5UA1Sko5HkAUY2yTYRjLtNqv0SClqq1LSV6fC5T1q9VF9kgp47UD1UgAYMqBamVzOVAFqFettt8nI8mtbkSn1fZbCNFZSjle73kqGWOZDodjDWOs/HReV32gGvcE/727/z1fCUvNBE2YB1vRTERcSjlACJEOFTVQogc71rUl5bSuV6MC7N3NAQXrPV8dU/Xuu+86P1i4sFPekSPRSSkpx664+urVd955564Axw/XDgmj9fHyOeeZWpzRYsQPpwO6Xk2wDChYHbis97wbyrmCQ/VBnFBZ6Wt0LFbAz6UmXhugSPT+AFKISNKpq9JTAdwKRTYfCvKl+iPk9dkTTzwx5LXXXuvs9Xq3A7gSetiZiJ4O1Tlt2LDRumET6DZs2LDRyuByudhTTz015fvvv39069at6eedd96h+fPnLx06dGiddkraTvtsTRaVagJ4nT+ZTTW5S3Upyc1p2UD2ddaMt6AXAacbpFTA3bVNYR+ogYQVeiChyUMBjSTJrQ2HFkOSNwR6IGGAzknvCsBlUVeclvuGVL5eXSS5NV/vOGomxK1qlGbP1yMihxBiqJQyHWpKO1/bu29qLe8/qm11b802NFX8VptSs7nsb2FX0ZR7XqtURkgpR+lz5uqmUEiaGS0Ffk1O/4aP1baxDGrNrXbrJzSXdcM8jYhGQNmLH9bq/q0hbApZVenQTaFdRNRQZcQMqPf4W8G+Pj9EAbgXwIcAdobqJJpAn0VEKxhjwwFMI6K/N+WYjLEYAA8DGAaVI5gC4BEi+mvTr9iGDRs2Qo/FixcnLVy48KFVq1ZdHxsbi5tuumnD9ddfv8LhcATcu+uBz5HagSqWKRvoZZzz/ACvjTAjqqDrJcve3aqqrcSJSnJz796kfUxLBRFFaWX/GChl/2bt5NXkoS69f4yxELUnI8n988hbBEneEEgpE4QQY/VAAteDHSuDsY6nAiIyLIShP0luzWi2Doa4Tvc9rwdn03UvRDLGNhqGsZpzXtCsF9IEUG2r+2S/tbdG4lnrVWvNWtSUoW7dC+klpRyt998evY6ZrWkdGwtdY8QQUSDLdWs0m1mvBnLgcluOZ2iniVFQA8FuvY5rT3VgqwE/g6lK70ZEEQAqLLFYDRkmaQdgLhSpHWql9gyodX07VCd4/PHHh7311lthbrf7EsZYHIBZANYR0dqmHNeum2zYaLuwCXQbNmzYaKVwuVzsueeem7BkyZLHNm/ePGHy5Mk58+fPX5aWlravru+RUiZrIn0olHIzG0ApYyyBapTkZhFgWk8HUkS0iqZDqCClbC+EmEhEZwFwa+Jyzcmm2c8Ukryh0FED40k5JLi1uiIzFOoKIgo3SXLUNB3MyXxr08GMGPAnyQtbIqGqmxk9pZTpupnhZirHM/N053ia14caFb+/KsLa5JSwqCL8lPwhtynVgx19pJRj9ICMmymb/LWt1SbfMqBQ3fAx1x9KMW7CA5VtWN300c0eV2OVJaTUPmZTqBeUaspsCoWkuUY1qvQeACKp4ar0S6GavQtDcV0WxAC4G8C/AJyg7AkWBg4ceENJSckVRLQ6WMdkjHUHsB7ARqhrvxHAo3YjyIYNG60NX3zxRfxbb7314KpVq24MDw8Pu+mmmzbceOONy51OZ8DPOVJRTiOklGdD7dGPMMZyATgtexhrPrNpPV2LKNeq2sY6pLQZkFL2j9DK/jjG2C7tiFavipFOjEpqsyR5Q0AqaiBND87GMsZ2c85XhsKBipSqNt464GohahNQO7LHvN/966YW6Z5ARNHaaWIU1P5srybS94S61mjg9QUacDX37v5W9/6W3+a9H3K3P8vgbBrUoOh+HS22qykk/emEpVcQyIHLjCMk1ESzWde+4FTqVSllkmUAORJAth5A3hGKZxfVqNL7QQ0gN1SV3h7ALVC55KGO4bsM6jN3YahO8Oijjw5/5513mNvt/k0wj2vXTTZstF3YBLoNGzZstHK4XC72wgsvjPnuu+8e27BhwzkTJkzIvf3221cYhuHbvXt3l0OHDrW/4IILfMOHD49EDXFirWgJgAvAfs75MUvxVdJaC6DmgpQyUdsUjoCaZs8yDCMTgMPa8LE0HQKR5NamQ6GePG4zJHlDoNdxrF5Hrgm3lY0l3IjI6TecYCXJraractRNkrdaKzq9jmP0OjoZY9s456uaw5bcosIKpIqwqvj9c96sTc4W8bzR6zhar2OEblJmtpTmmhVUt22j2fAxXUMkaoZyrKqIAoToeaObQiP1OkYBOMA5zzIMY1sIm0KNUaVfBmWv+E6wr8UPsQDmA/gAwO5QnaR///43lZaW/qap6om6wBhLBnAMtpLChg0brRjffvtt7KuvvnrvqlWrbmGMRdxwww2bpk2btnXnzp2d9u/f3wlAzG233eaxELURfodwAzjEOT8Mlc9sOhG1GUvoUEAPJAyWUk6EUuUd5JxnMMbyoJ2fqEbJb+4fA5Hk/vv2ZneBOp2wrOM4KFIrTzt5bWnMPppqu85ZB4z9B+oFApPk5oBrsH/EZoFex7P0QEInqOHRNdpZLqS1INXYflvrVXPfHkjFX0vVHGyr+6ZAr+Mg7TTRBcBxTQCvC8VAfFNBykGhLgcu64BCOQI7cIXkeUPKWW6glDINQHcol4b1hmFkhWqQm4ii9QByd6gB5PJ6VOkdANwM4HUAuaG4HgsuhxpuCVl99vDDD4987733fG63+7JQncOum2zYaFuwCXQbNmzYaMVgjHUCMApAHwB9IiIiRgohhnq93mpFRHh4OB5//PHS2bNnH4FfzptWr47X08M+xthqh8Ox6kxWSjQEftPhSVLKdlDqx3i/l9okeSOg1RWjdDMjRhOXKzjn2eZaUY2K32odaJLkMZbDuWEhyaHXXytrq5r9h2tGaLXPcL2OSQCO6FzqJhGXfmvv3/DxH1CoSxXR4lT8dYGUjetgnfnXEUCRbgqtb+5nJBGF+a19imXtA9k2+jd8TttAlG6umU2hHlBNoQ26KVQYonP6q9LdWpW+DTWq9MuhmmX/F4prsCAOwF0A3gOwN1Qn6dev381lZWUXE9H6UBzfbgTZsGGjNYMxxgFMhK6ZDMPoHxYWNq6qqqoDEVVvyIcMGSK//fbbQ9Z6Se8ji6SU/bQiPRVKcfmLYRh1RmjZqKUkTyKiZCllElQedQcAYX4vt8aDnbEkeUOga/heOhKrD1RE2yprvrde+7g6SHJrNJvpAmWNSjJ/B8db2vBoMKHXsat28hoEZUu+XqupT5m41AMKseQ30E0n2n4L1B1T1ap6BdomfzQRDYEaiN/KOV/THIPcVljWPpDlulVIUj2ggBMt109br0BKmapV6cOgarx92slrZyhqOaqtSk8hhXwi2klEpl17RwB/APAagLxgX4MfroBS478bqhP8+c9/Tnv//ffdlZWVV4bqHHbdZMNG24Lj5C+xYcOGDRstGJcBeAnKxmtPZWXlHgC/dOvWzT1y5MjLs7Ky+nXr1u1Yjx49VoaFhW2DmuCvBcMw/iulXCaEGE9E47xe7zimrLRXtMTJ4eaCP0lOFvtABFaS52hyJg7KujgSyhZuuWEYOc3/E7Q+MMYqHQ5HhpRytc6pGyWEmCOEqIAiZcNRe+2rUKOqPWApfM9ou0zGmMfhcKwhokwpZR8hxFgp5WVSygt0vndWXe9trYqw3vdWktx/7c1mw17U2H4XtpUBBcaY1+FwrCei9VLKLlLK0VLKqVLKKUxloq7hnOcCwPvvv9/t4w8+GFF09GhicufOBVfPnr1h5syZjcqIsyhSAg0oWNe+EspyvYAxttNvSCHkto2NBWNMOByOLQC2SClTdFNopM/nGw9FQKzlnO8IZlOIMVbBGMsioiyoRn0/wzD6E1F/qlGlMwT4TAwBzCZpqGMIOFQj1oYNGzZsBMY3UITEISHEHiHEV+Hh4dmTJ08eW1RUNPHIkSPhF1xwwTaPx/NLTEzMCfskzvlmItoipRwohJgkhPi9EOKgYRhLOed7WxPZFUz4k+RUoyQ3iVorUV4MoBDARsYYEVFnKAVwiVZSr2+Je5mWBsYYDMPYxznfJ4ToLqWcIKU8V0o5FUqtL6EGu821J9QMKOxnjK21kORnrOucXsdDhmEcklLGaQI4zefzjWWM7dROXtmB3tsU2AUqybJ3tzooFOm13u2n4m8zAwqc81zO+ZdE9J0e5B4thBgihMjT6v4tjDHv3r17o5599tnh27KyejPDkKMmTNj5wAMPbExKSmrU+56IIutx4DohIowxtsOy7gWshcYMcM6Pcc6/IaIftLp/lBDiSiFEGWNsnWEY6zjnJcE6n16DbMZYNhFFAxjIGOsB4ByqUaUX65c3V90U0vMIIeyayYYNG42CrUC3YcOGjVYMxlgiVCMojwI80J966qmBX3311V+zsrJ+NWTIkMJ58+atvOCCC7agjk0pEUX5fL6xRJQONTmcZRjGcs55aWh/ktMDC0luVTQHIsm9qJkO91dFnFB8acXlUCnlBH2sbMMwMs7k5po//MhCfyV5PGqmw31Qxa8TirTdwzlfZxjGUbSyyfzTCT3Nnq6n2ZkmXvcB4AFUEda1r0sVUXEmrj2p7MQROjsxHkDO66+/nvv6008PGOXzGWmM0WoivsHh8M65447/3n333dv9vh9EFONPkJtNN9SQrT6caLleoIcUWv1wCCmrwkF6HbtCvZfNplDxyb7/FM8ZKaUcSkR9AcRKKeOhFPtvo/6s9KYiEcCdUEr3/aE6Se/evW9zu93nENHWUBzfVlLYsGGjtYMx1htADhGdYH+8fv368KeffvrWFStW/PH48eOpv/vd77bOmzfvl7i4uLJAx9LK1X5CiMlQBHCOJtJ3tcX9kR9JXm2zXh9JbqmVzLopoJJcStleCDGBiAYDcHPOVxuGkdkW9jvBgIWoDaQkt+YzA6pWCoPaT+ZxzjdyznezNpQHH2qQcqAaIqUcC+U2kc8Y2w51Tyf61ayBXKD8VfxFZ+KAAhExKWVvKeUYvfeuzM/P33zVpZd2MXJzEy6Gmgj+L4CY3r0PLPryy8Xx8fE+v2MEcj8zLdcjLS8tRWAHrhYTEdYU6GekqUp3MuXSt1a/t4NO6ujnfQ8iGkpE7Uk5eyUA+BTA5mCfzw+/hRqAeD9UJ7j//vvHfPTRR66qqqrfheocdt1kw0bbgk2g27Bhw8YZgBdffLHPZ5999tfMzMxLBwwYUHznnXeunj59+iYoYvIEkLLSTteFY5i2MlseKmIjlKiHJE9Cbcv1RpHkDTw3k1IOEEJMhGqu5XHOM7SVdpv/AKaafL1qkhw1k/kJqE0WFlnIQuv6lzLGIKVsp/Pmh0LFDawzDGNVMCew2xL0/i46gCrCbDpwy8tLAeT6KSLMbMM2f5+eCoiISyn7FhcXp19+0UU9ZxQW4uGoKLfh9VZKIeTdFRXRXyclVSzJyPglKioqkWpbrlsbncVQanJ/y/s2o0g5GfR722wKhQPYo1Xpu0+l6UW18w1N+0bzvrdGDXg45yWMsWIiOkbKqjBQVnpTkQTgDgALAYTM6rd37963u93us4lo+8lf3XjYjSAbNmycCcjMzHS+/PLLN2ZkZNzrcrk6zJ49e/sf//jHnxMTE48Her0m0nsLISZBOZ4c1UT69tb2Od5IkrwEtaPB6iXJGwIpZYJ2RBsBgBhja/VeP+DatzUQUYRJkqMmF94csLQStSZZaF13k6j1aQJ4uM5JTwRw0DCMFXq4o1Xdk80Fqu08Z62d2qH22nsBHGOM5fsR5S3SBaqlQEqZKIQY9cILL4xesmBB2KfR0d4uUlYyr9ezyeczLvV4In5///1rb7nlljLrgDFq92qqYKmZUNv9LKS59S0FpGLaBusB5I5QmfPr9FejBS+WfoHV7t6sWf2jBqq0gKFMq9K348Ss9GBgFtQw/79CcGwAwD333JO+aNGi/KqqqjmhOoddN9mw0bZgE+g2bNiwcQbh5Zdf7vHZZ589umbNmpm9e/cuve222zIvu+yydaibSA8XQozWBXgEY2yTYRjLQpVbe6rwI8lr2QeifpLc2ngImY2Xbq711ER6L6gs5eWGYWxkrSgTOhCodsabP0meiNr5ev558OafDSYLpZQxQogxRDQaQDhjbBvnfIVhGEdC8gO2cPg122rl7EGRkSbMRqfZdCgCkCClPAuq4Vui7fWqsxNt1A8iMj7++OOznrr//ku+j49n3cPDq6ORtnk8uLyiAi++9x6GDx8eKBO+utF5Gn+EFgWqyZwfBTVwVGppCh33e62p5K9FkFueO9Z8Q2uzzVTxuxhjXiKKpJqs9CiqyUrfDhWNEgwkA7gdSukeCoIeANCzZ887q6qqxhLRrlAc324E2bBh40yCy+UKu+eee36fkZFx39GjR7vOmjVrx1133fVzu3bt6hwmFkJ014r0ngCOcc6XGYaxtSUpIAOQ5P652PWR5FaiNmRqZiKK9vl86Xqv79T153LOeUGoztlcICInESVJKQOR5NYhP3Pv6E+SN5io1YPc/YUQ46Hcfgq1Tf7GM5HspRr3s1qKZpxI1J6QjQ3AS0S99SC3Q9efqw3DaFRc05kKk6i9cNKk2Rfm5KT+NTmZm97hAHB9QQHw61/jf598UqAOBy6coe5ndUEI0UlKmUYqc97BVORAlnY8rNVXIaIwy33vP1xs9gsIQLFl7c36qYAxZpLz3YmoPxTJDlKxWDuoJis9GLhKX8uHQTxmLcyfP3/c4sWLD3s8nhtCdQ67brJho23BJtBt2LBh4wzEa6+91mXRokWPZGZmzurSpYv71ltvzZw5c2YW5zxgM0QTG2nakjyaqfzfZZzzY811zY0kyc3Cq9lI8oZCStlR2xSeBaCcc75K2xS22Nxoqm07bSVrzfX3z3gLRJIHNV8vgLrigGEYK9uiuoIC29eZDZ9AzTZ/+8DC+ohafU+mk7LOlIyxjYZhrG4Ljcqmwm9AxN9yPXH16tXs3htuwJfR0ehrGAJEBM6N9R4Pu8rtpqfffDNz4sSJP9lDCY2DvifTzEYlgDzG2DFSueVmw8dU8ldnS6K2mr/B+Ya6HupCRAMAtNPPvGCp0lMB3ArgLQCHmnisOtGjR48/ejyeNCLaG4rj240gGzZsnIlwuVyOBx544Jply5Y9kJOT02vmzJm75s+f/1PHjh3rHCYWQnSRUk4iZVtcqIn0Tc1FpAfYt5+MJPd33jKtj0/rkJ9WW6bpvX4sY2yHdvLKOZ3XdTJossqqZq4eMAYQY3mpG5Z6CepeMf8d1LpQCNFZSjleDw1WMsYyHQ7HGsZYeTDPc7pRx2C3dcDSqqitK6aqzogwUuKC4VLKdH28HE2kbwvlUElrgd+ASDVZC03Uzpk1C8M3bsQTSUkSUkowxsC5cU1BAdi0aYUvvvTS4jN1KP5Uoe/JoVLKNADtAZQByIEaBI7HiQMi5nPHSpCbz50G3cOkLN3NAeRIIqoIoir9Gih3wo+aeJw6MW/evPGff/55tsfj+UOozmHXTTZstC3YBLoNGzZsnMF4++23O3z00Ud/Wb169ez27dt75s6du/bqq69eUw+R7tD5vxMBxDHGtmubwrxgXE8AktzacDgZSW42fE47Sd4QSCmTtE3hcChLctOmMGDWYqhBNfl6tVTMlmabv+20vyLitOSMUY1N/jgodYVLDyW0KnWFVkVYowasJHmc5aVVqK0kdwWr2UYq33uUlHI0gGgAe/U9ecIke1sDKSW/dd1TLM8es9FsErUF0L+DioqKovMmTJh2XlFR7ItRUeVhjKGKCLe43bFbevYUX3z7bbhh/H/23jw6rupK+37OuRo8yLYmzxM2GDBgA54nIIQkkABN4oSEMITQhI+AwYDdJJAATjehSUL6JZ025CVZaaf7+7rzJpCXTHTIwOBZg2VZni1rsmxLtqySLcsSGuqe/f1x9lWduqqyNVRJVdL5reVFUqrh6t6q0jl7P8+zHZfdU4VSypMD9TsmKlzszDTe97lR3vuALnjWCT37ryYeTjgKd6UPJ6LWPrrSxwF4BMDPAcTNqTR9+vQ1HR0dc4moKh7PbwtBFotlKBMIBJznnnvui5s2bfp2VVXVrM9//vNla9as+XDq1KlRxcQsBruexVmN3PwtjsXfrChN8uwIaxcgcpPci1tP+CQcInK4QbQcem1Q5TjOFl6fDtgx+UXdxrmPtG43G7Veo7zfxZUck7+EiOYBkLw+3d6fovi+EmHPaqaf5aCrsNurF8RsVBLvP2e5rrsYOl3unJSy0HGcHUKIWCUYJSTGnjWSsHuUcdcW+ITd69atm/j+L36x8M309NarUlJcAHi/oyPtIaJhz//oR+2f/vSnR0KLEgo5vSPhv5/6G2PP2rlfIu0mN00NHk0AjgghyqWUASllvRDioxgeCwBMI+1KH0uaU9Q3V/q90CkQv47Vcfp5/PHHV/zud78r7ejoeDRer2H3TRbL4MI20C0Wi8WCN998c+yGDRuez8/Pvz87O9t9+OGHd95///15UsqImxajkHEdgCwhRKmUclN3HAHcJPeaJd1pkptu5qRqkncHjiRfQjqm0BFC7OI5dXGJ3CXBYgAAIABJREFUySei4V7kt69J7o/8PovITfK4xjb2BXb8LDXcFQUpKSmFieKu4Ebh6PO4Irw3dRDRXRFxj6/jz/eVSqkl0PPVAhzvvksk8Yw56r6T35stGSlyvYtA5I033rj4te9///Yp7e3p1xJhhxA4MWxY69oXXvjtvffeG1BKzWdXwCgA1VwUOpCon6N4wd890Qo+3qgHzxHUxU1ORDlKqQWkkxKk0FGFO6SUlfEQeFB0V3opEfVklvl4AF8H8DMAcXPVTJs27R+CweBsIoqpy10IcQu0oGY0tAjg1wDe5B+/Q0Q2XcFisQwZAoGAXLdu3Wc3btz4Qnl5+RV33HFH+dq1azfOmDEjqphYKTXOdd3r+O9XE49x2nkhoafRJA9L3rpAk9zfKE+KJnl3MESzK6BHvZzgc7k/HgLeCOJW/57VW5B7I8LMRm1DIsdOE9Ewdvcvhnb3l0kpt/GaaqAPD4B21lK4m9ncs5pzyc/CECdcaN0ea5RSYznJ62oAQgixhwXISSuapdCoh4gJXAg5+b09qz9yvSFSo7auri7ty5///OcbKyunriASrQDypaSLrrnmwP956613UlJSZrmuuxDAxdCfnWLHcXZIKaOOzhiMGN89nXHrxp51pHFXb88a5iYH0KqUmstjsXKh0y12sMEgLgIPCrnSZwBIp/CxWD1p2t/H938rHscJAI8++uh1f/zjH/d3dHQ8Huvntvsmi2VwYhvoFovFYunkN7/5TfYvfvGLb+Xl5T2YkZEhHnrooeIHH3xwW0pKSsQCDxFJnll7HfTivJwd6ccNZfKQb5J3By5kLOCm5Qih3f1bpJS1vXgus+CQ7Ss4DDfueg7hinxztmHSuLf9KKWyWJRwLXSjrYTdFXGPJOd11cgorghTGa4Q3RXRlAiObyKCUmqqUmoJb4jbuZCRn6iFDBYpjInSJM807up38nuzsRt6IxLYuHFjzn/8x3/MqTtxInPi5MkNf//3f79n6dKlp43jklz0XQRgOoBzQogix3GKpJRN0Z85uaCQI8vvJs9F+HfPWXSNWw8IPerhvO99/q70ikJjAZyWUhaxqy9eRaHh3EifAT0rvdWIKrzQa04E8P8A+CmAHn+fd5epU6d+w3XdS4gopk16IUQlgGlRfjyD+h5xb7FYLElHIBCQL7744q0ffPDButLS0rm33XZb5dq1azfOmjUr6newUiqHG+lzAbRIKbdLKQsBpNkmeffh9ekM13WXQzfaTkspt7HQs0fngUIpOH5xsbduNMWtDb51e9LvWQ3R7DJowd8JnpO+tz+azxRKn/Ov2/1x913czMb7PyH2rEQ03HXdeUqpRdDNsyPcSD+UCPu6SBBRepQErhx0TeAym+S93rOePn065bXXXptdmJ9/kXQc9bEbbyx75JFHStPS0jqfh78rF/BePl1os0aBlLIiUc9lT6HwFBG/uNgU1rfDECcglD4XuNCelb8rp/Os9CsAQAixnwXI1fH43qK+u9Lvh65R/SbmB8d8/etfv+Gdd94pCQaDT8b6ue2+yWIZnNgGusVisVi68N57743+t3/7t2e3b9/+UFpaWupDDz1U8tBDD21JS0vrXKS3tbWlpKamZnqNWiK6GDpCO833dGaT3F/waUrWgkO8IB2TfzXHFGYBqOBGepgjgPR8vTBHhNEkN5XJXsEh0ny9pHUTdwdDlLAYQIbQsc/bpJRVfX3fmQUHdHVFmE7+RkR2RfR73H1fUEqNdl13ERHNBzCM3b95sTiXPYUiRzeakeuem1lBf+d0Rq6Lbsw3jDfsRFvIBfRUIcQBKWXhQJzL3uBzpfgLPmaxuR2hJrm/4NPnYicXhaZxUehKAOBzuUNKeSSORaGeutInAXgIwBsAYjLuJBJTpkx5Rik1nYiS1vFksVgsyUYgEBD//M//fPMHH3yw7sCBA/NvueWWI2vXrt04e/bszkaB67pwXTfDcZxs3jdNhm78ZkV4SrNJbu6bhlSTvLtwTP5ybg618BinHcKIR6dQAlSkJrl/LrZf3Or9NyHErfGC11QzeSTWJdBJCfmO4xSJPkbNk3bTmiIFU9xqCus7G4UIX7c39PUY+hNDNLsEujZyhpO8igfi94ggbjXPv1kzOIfoCVwDkppFRKmu685hUcJ46DpGIYtlkuI9wTWbnGhz4b27ATjj27PWG989sTiOEa7rXsOpaNkATrEAuSRe55Iiz0q/kCv9q9BC6/8bj2MCgIceeuhjf/rTn3a4rvsP8XoNi8UyuLANdIvFYrFE5fnnn89+//33X9y/f/89rusOW7hwYXtzc3PqsWPH0kePHi3ee+89764dABr4H6Add5kAaqWUHziOczgZGkOJBHV1958FUA9d5PHPGGuFr0mejAWHeMHuiqvYXTEO2l2xjWerRW1iU/TI72yEFxyaEV5wMItug6rYyYWMuSxKGAvgJBfY9sT6d41QcDALPmZ0oylSMP+dSeRiJxGls1hmIfRn/BQXhUoSQdxCRGnnKfh4Qqkuc+GNgk+/ObLY9eMVhXKgEwW8olDMZv35XnMYhaIKPVe6VxQyx0ZMBvA1AD8BUBePYwGAKVOmfEspNYmI4p60YbFYLJZwvvGNb4jKysqVu3bteqGqqurK2bNnN48dOzalpqYm7ejRoylvvPEGPv7xj3t3PwsgAN2wyoT+OxEUevTQtnj93RrMuK6brZS6gQV1BOAkdINmDHST3EyAOhOlSX7BFJyhAEeSL2Whpyt0+lTe+dKnWKTgiSsjjakyRQrefjVhxK3xggUeS0iPb3CFHtWWz79vzKDwMWGRErgiuZn9CVxtsTymWEKhVLRFLJZRQojdjuMUJEJUPoUnoOUa5z8XOo3Aw0xSMJO4GvpLpEB6FMZFPBbrcuhzuVdKWSSlPNZPrnQQUR0LkP2O7AcAnAbw25gfCPPggw9+/N13392mlHomXq9hsVgGF7aBbrFYLOdBCLEAwPMAFkJvwKsB/DeAHxLReYsbQoixAF4BcCv0fN0SAM8R0ftxPeg+IoTIAPA2gFnQ8UOCb+8YN26cM23aNDFlypTmOXPmVKxatWpnJCc5b3IucV33emjldS1HuydshNlAwapwcya82aQ1VfkEfS3aAVQJIQ5KKU/xrPSEnK+XaBjuimXQzp+zQogCKWUFgIwIBQdzw+uP/A4kQ8EhXnjnUim1mIguhX4PFjmOU9iTSHLDlRKp4GOe/48Q3iT3ig5JPW4ACEWRKqUWciGjQ+ixAwXxHjtAoehSs+CTi64iHVMkYs7ZOzNQrpRI8Ln0ikKzoYtCXlTh0Ti70i8DMJ66utKnAvh7AK8DOBXzA2AmT578bSIaR0SnL3xvi8ViiT1DdN/0OPR3/CUwIqfT09PbZ82alTZ+/PiOiy+++OSdd9656+qrr66O5CRXSmW4rruMiBbwcxZyI70Zlk4oNCYpbL9k7JtSIzzsFKfTHOU1e1IlQA0k/L5cxO/LYbye2gmgw9u3ItQkN8+/GfntH1N1dijWApRSGUqpBTx+aCSAMm6kl/fkfBDRcCOBKxfh598UiXQRKfS3uDVe8Lmcx+dyFICj7PA/EO89CRENU0qFNciNfauXgOaJRMLExVw3SChxFJ9LT4CcCS2M38HC+LjUN7rhSn8Q2jTyu3i8PgA88MADN/35z3/eRETfjtdrWCyWwYVtoFssFksUhBBXASgEcADAS9BOgeuhC0N/JKLPneexaQCKoBtA34Qumq8CcBuAm4hoc3yPvvcIvav6bwBHABwGUMb/rS0pKUl/+eWXV2/dunX1Rx99lHX//ffvffzxxzdlZGRELPBQaEbd9dCL5Dop5WZ2/g6ZP0DcJPRUyf4muakKjzQTvlOVr5SarJRazpuOczyjrmgoNnB7gk+V753/iQAmIDxu3Zxv6HdFJJ1IIS8vL2vz5s0TJ02adO5LX/rS0ZSUlLh85pRS2VxguxZAChfY8h3HOQZ0mfHmL/iYrpQgIkc3JlzBIV5wVP580lH5IwFUcSP9UF8Kvlxwi1TwMSPvvfe/V/CpN85/0iVZENFIdqUvgP6ereOi0O44FoWGsQhiJtiV7rpuI4BlAF6DLgjFhUmTJj0P3UhojNdrWCwWSzSG8L7pPgAroPdK3r6por6+vvVf//VfF/3lL395saSk5GMrVqyofeqppzYvXry4LNpzEdGIYDC4lIgWAZAsTNzaE2HiYIDXLNGa5Oa6/SyM/ZLh5DwNIIXHOC0FMJLXplsdx4k6o96iIaJ04hFtAHKUUrkApkCLYszNkHf+I42pShhxZXdoampy/uu//mtGc3Nz6h133HHkkksuaYnH61AoFW0xdGJfgPdMnelTFD6mzS8uHm48nXn+/Qlcg14kQjoq/zLXdRdB15ma+TuzSEp5tg/P6xBRlk9c7LnJRxh3Nc+/KS5OuiQL0q70i1mAfCl0Isoex3F2SClr4/SagHalX4rQWKw6pdSlAI4B+H08XhcA7r///k/89a9//YCIno/Xa1gslsGFbaBbLBZLFIQQ3wXwLIBLiKjSuP1/Q88zjVqoFkI8CuDfACwlogK+zYF2UzQR0dJ4H388KS4uTv/hD3/4yLZt2546e/bs2HvvvXf/6tWrN2ZmZkYt8LiuO5Ub6ZdAbxY3s7p1UGzwqGt0mqfKz0bk6DqvSe6fr3fB11JK5bJT5WoA7exUyR/KThXDleI//5FU+ae98w8dnzmeiGYBSOcC27ZkLrA1NTU5X/z85+8o3LlzjnfbuKys0z/+yU9++YlPfCJuDlil1Cgj8nEktIq8DbrYYEZ+e9GZ/oLPkHSlRIILbLN55t9UAGc5krwo2uecwmcc+t3kZsHNi7w3owPrB+v556LQTC4KXQZdFNrLrvSaWAljyHDzc7F5Ogt1MgA4QoidAPYQUVVMXtBAKYUpU6asAzCKiM7F+vktFovlQth9U3RefPHF+e++++6LO3fu/MTixYtPPvnkk1uuu+66Uug1UReIaJjruouVUksApAod+7zlfBHayQYRpfOavXO/ZKzZzTWLOZfZjFzv1pgkIkrhcTnLofdjFSxKqEg2YWwsIT2mymsS+sdUZRh37Yycht43DSOii6FFyA0s5i4RSZwG9frrr8965eWXP9f00UfDASBFSnXr7bdv+tnPfrZRSnmhh/cKpZRQSl3B6/xp0PWBc9D1gkhj2vyN2oZkPuexhscOLOTaSKrQSX2FUsrKSJ9zQ9wdSVycha6R9/XeZ0BKWc/nf8DHbcUDpdRopdS1Sql50KK2Gt6D7onle460m9/bs04AcBERZUHXYxoAbCOi/Yg+K73X3HvvvZ94//33/0ZE34n1c1sslsGJbaBbLBZLFIQQzwH4R+g5PQ3G7d8HsAbAaIoSRyiE+At0pOsVvtufgXZlTCGiuKg5+5PCwsK09evXf23r1q3/UF9fP/Gee+45+MQTT3yYk5MT1QHnuu4kpdT13Mg4w430kmRQqvNmaxRFbpKbTs6wJq2v6BOzJpXRsJwP7VQpdhxnu5Ry0Eb4GgU3b7Ze55w9hLtSzLnYfldEF9EG6dne17BTJQvAET6XpcnWVPzSl7508+b331/8YyLxeQAHATwiBB0bObK5eM+eH40aNarXnzWfKt/vijALbp5jeRiANiFEqZSyQEpZmwyf9URCKTWBi0JzoQtrpVLKSiJS0G4I7xqYaRbeyAHTETHkC25KqVFcFJoPXRSqNYpC3SqE8XeQKVCIFN9opimcFkKkCiGGCSFGUvRZ6b2mvb1dXHTRRS9Au96HRFqDxWJJLOy+6cJ873vfm/vOO++8WFRUdMu8efNOPfnkk9tvvPHG/YjeSE93XXchr02HCT3zdzOPb0p4iCiN1+meuM9sko807tqMrvPIvSZ5TJpURCRc171CKbUCuvlby6KEA8m2zu8uFJ6A5l+zm2PCOudiIzwBqkFESSByXXeyUmoZ6VS0VhZzFySbmDsvLy/rs3/3d499UinxA0BkQccFvQxg9erVb7/wwgu7e/vcEcTd/gQuM/K7DXrPJAHU8p5+nxAi6RLQBhIiSmPBzEIAYwHUCyEO8n8zKVxcbIq7Txvvf3PvlPSR972FtMP/EhYgz4Lez+9mh3+35s5T+Ki2XE+wAL1/Nf8GeG7+eiGEK4Rw+Hp5Y7EOUddZ6b3my1/+8ic3btz4LhG9GKvntFgsgxvbQLdYLJYoCCGmANgBYCOAZ6DjBD8G4P8F8Asieuo8j60BsImI7vLd/hkAfwBwMxH9LU6H3u8EAoHUp59++v4tW7Z848SJE9Puuuuug0899dSH48aNi+qUUEqNd133ei6WneVYvZ3dcRPEE0OR3FnsQXiT3Jzvdgbh0YHefxsjNWnjeMzDucC2GMBwIcQ+Lgp1a3OTaLArIlJ0XaSCm6nKN69Dr95HpN2ql7uuuxTa+RuQUuYli7uisbEx5YrLLvvm08FgirkjPADgCgBPP/30W9/85jf3ne85SDtpTaFItCZttIJbQHBEtlJqnOu6i7n5K4SOg8tL1vdmf8FFZ8/J7F2HsdAFB8e46znoQtsp4/zXQ8cYDsShJwX8OZ+llJrPRaEOoyh0gkJu8s5GuVFwM4UiTdBFOa/oU2/8DSDfawLAJNIR7xP4NbxZ6VV9+X2am5udWbNmPQcgnYgGpSPGYrEkNnbf1H1eeeWV2X/4wx/+qaio6PY5c+Y0PPHEE9s/9alP7UX0Rnqq67rz2UU9UgixlxvpcUsV6i7Gmt3fJPf/vfwIRpMcvH8y14z9dLxQSs10XXcFgBnQLuqtySLm9sP71tEUPiIs0pgkLwEt0piqXq8ZlVKZrusuIaJ50GLu3SxAHvD3Zne47777btr27rvLa4iEmc19K0B7Jk48WbxnzxsXeg5jzd557qOIu726gf9foxCC+HM+l/fzY6HnUec5jrN3oOsjiQyvp8f41uw5AMYhvG4QhF6z1/rExaeT8bPfn/DnfB7pcW0ZAI7xWKx9Qogghc+G94uLve+gDhiR9wC8ufCBSEIpIhpOoVnpI4joI0OA3KcRC1/84hc/tWXLlj8S0T/35XksFsvQwTbQLRaL5TwIIS6Hnr9zCd9EAH58viIQP64NwM+J6FHf7UsAbAPwZSL6VRwOeUAJBAIpzz777D2bNm165vjx4zPvvPPO0jVr1nwwceLEqE4JjiO/jojmAGjm+Owd8WxW8t++EcZ8PbNJbiqSgZCT2d8kT7j5bhRyUS8DkCmEKJNSbpFSHkm0ZppPkewvNow27uo5ac1Gbb8U3FzXnaKUWppM7oqSkpLRN91001N/BHCr72djhKBb7rzzvddff30r0DljMuzcG9fAH3kfKXK9WyMHvNdyXXceRxWOhnb45/Fs7yG5GI3wGTCLDWZ8YzNCTdoA9BztkUqpywFcCu0IKHYcpzBZXGmJAml333SOKZwB/d0fhC72eAUfz03eORue4xsjFny6+brerPQZ0EWhNiI6opTaj1640s+cOZNyxRVXfBtAChEl1N8li8UydLD7pp7x4x//+JK33377xcLCws9efvnlZ1avXp1/66237oZee3WBdBz5teyiHi2EOOA4ziYp5Yl4HieF0ociNcn9a3avSdu5b+I1e8Klo3Aq2gpe55/jOPKi/mzodwdj3xptTJUp7vYnoPXLmCTSYwfmc/N3FO9Bt0WL0E4UPvnJT35+9K5dV35IFHaQLwF4OT297cjx498DOtfs0RK4zDV7Z+S9CB+VdLq7TXBP5KGUWsIizxYhxA6eRx11XN5gx7dv9QTGuQhPAXQREnd7sesfEdF00vHuIwFU8Z7pYH8aHpId0o70bKXU1fydmQP9t8pF6DsI0G5yv7i4nusGPf4O4u+/KfyaYykGrvQvfOELN2/btu23RPT93jzeYrEMPWwD3WKxWKIghLgUwAcAygH8L+imxWIAzwN4i4i+dp7HDslCkEcgEHCef/75Ozdu3PjtqqqqS1euXFm2du3aD6dOnRpVja6UynZddwVvblq5iFHYlyIGq2E74wKNzW42dEyaRxMiN8m7vdlNJIhIuq57JRfYxgE4xo70fo0jp9Bc+EiuiEyEN6iiuSIGPLpOKZXF7oprod0VJeyuqB/QA4tAc3OzM3vWrKcfbm9P/1/G7UUAFgBYv3595cqVK1OgN72m2cL7DPgLPhEj73sLb74vd113CbTD/4yUssBxnGIRJSYy2SEir+iZa3wGvIKPv0nbWfDxPgPnOy/sCFjAzp/hAMq5KJR0owfiBYW7ycPcEQh3pjRBFz6HQUebdggh9vHforo4HRsQcqWP5wJtPWlXeuUFHt7JqVOn0q6++upnAUiymzuLxTIA2H1T7/nJT35y0VtvvfWPBQUFd1588cVNq1atKly5cuVORG+kOxxTvAJAltBjcjY5jnO8t8dgiPrMBChPWDwGofShDnRtkgdYwJeU6TdKqRzXdZfzHrSDBbN5/S2Ypa5OZlOoYO5bvbhjc0RVQoi7+b15JYu5xwM4weuofQN9bJF48MEHb/jb739/wzEikcm3EYCPC0GB2bOb//Lee7XUdS52p5MWXRO4YrqX4frIYiK6BkAKr0vz+/JZT2QMsY7fTe7ft5pNWnPv2iUBynxu13Vnc7z7NABNUsoi/ncu/r9dckAhN3ln3HqEfWsHdKKCgP77kAo9FiufExPi8lmncFf6cNKu9CPUQ1f65z73uVvy8/PfIqIfxuM4LRbL4MM20C0WiyUKQojfAFgGYAYRtRq3fxXAvwO4gYg2R3nskIoijEYgEJDr1q377KZNm54vKyu78o477ihfu3btxhkzZkR1SiilxnARYx6ADl6I50dzLnCxIZuLDf4mubnR8uK+I83YS/ho7t7ACvZZrusuBzAdOiprK8/7jcnGhrrOd/O7IvxO5kiuiF4pkvsbdlcsYHdFhhDisJRy+0C7KygUXZdDRDnPP//81T//2c8mvQDgC9Az0J92HMgpU+j9TZtq0tLSOos+XOyJ2YzJnsDOn8VEdBUAVwixy3GcfBZOJBXUdeyAWfAZbty1EZELPn1yBpF2pV3JDv9JABqllIUsTOhTzF2ywAWfsLj1CM4U001ezwXPLm5yFs3M479DIwBUc1ThgXiJqogonYtCMwCMJKJW0q70A9BR/VGpqalJX7BgwTeIyDnf/SwWiyVe2H1T33njjTemvPXWW98pKCj48tSpU1tWrVpV+IUvfKFIShlxzc6C2auUUtdDrzfKHcfZ5DhORFeeb70YJjBGuLDVjPs2m7QNogfpQ8mGUmoUC2YXAHCETvfZJqU8HavX4PWi38nsCRXMyHvTyZx0+1beg85wXXcZdCJFE+/piwZaMGusF3MqKysnfuLGGxfPaW8X31UKWQBeB/BzAD/96U+bbr/99hOI7Obv72NO5/SJRdDN/GN8Pvcnm4uatIPYGxXmFxf7R4V1OpkR2rcG+voZ4FGCC0mPGHOEEAeklAVSyurB+v1mYgimIomLIwkV/I7yztqNIUyYD93YbuE9fVG8ktEo5Eq/HMA46qEr/Y477vh0YWHh/yGiV+NxfBaLZfBhG+gWi8USBSHEAQA1RHST7/YrAewBsIqIfhLlsX+GXtRd6bv9GehUsMlEFNe4vUQiEAjIF1988dYPP/xw3aFDh+bedtttlWvXrt04a9asmmiP4SLGMi5iKCHEXiHEUQCd88mhGyP++XqdRQbojVYD//+EiuPrb1zXnaqUWk5El0HPnN/OM+e71TglonSv2ICurghzvps/8j4uTuaBhDeKV7G7Yhy0u2Ibuyvi8jtGECqYbv6w6DrXdRu+/a1vOf/nv/87q911BQBcPH16zWtvvPHrBQsWNMbj+PqCUipDKbVAKbUA2hFcxo308kQSVlBozmSOUWzIgS42ZBp3bUN4dJ35OYh7ogULExaxMAFCiL3cTE96twqFx96HuSMQ7ib33Fn+ok+PhAr8Wb+c35sXAfjIKArFRehBXV3pgogCdB5X+pEjR4YvW7ZsjVIqNdLPLRaLJd7YfVPs2LBhw4Rf/epX6/Lz8+8dP358+yOPPLLj7rvvLjhPI124rnsFN9LHATgmhNgP3YAym+RZCK0X/SN6zAZtXOO+Ex0WzC5USi0BMFwIsc9xnK3djcrntUqnsNUn7jbXi+3o6mRuiIeTeSBRSo11XXcpNytdFibkSSnPxOs1STuZsyNcgxyErxfPbdu27dyaxx/Pqq6tTQeAkenp7Xfde+973//+9wvidXy9hYiEUupS13UXQwsum3iNX5Roglnqmqhgiou9cXne2AFvTJK5dzoX72Y2f9avZld6DvTc+UI2G/S7uDzWUCj2PpK42D+bvN4QKnjXoEdCBR7NOJ90YsIwABW8Z4pbXD5pV7o5FuuCs9Jvu+22z+zcufP/I6Ifx+OYLBbL4MM20C0WiyUKQoi/ApgDYCYRtRi3PwTgfwP4LBH9Icpjvw7gNQBLiKiQb3MAlABoJKLl8T7+RCQQCIjvfe97n3rvvfe+c+DAgfm33HLLkbVr126cPXv2scbGxrSSkpLpbW1tuTfeeKPwNWjN2WIugFNCiFMi5GYedMWGeMFFjOWkZ863CyHyea53C3V10ZquCLPY4Ln5I7kiki7yvrewu2ImuysuhhYmeO6KXgk2iCjVV2wwCz5mfKMnVPA3aTuj66qqqoZv3rx53LRp087dcMMNCe/qNoQJiwFMhE5MKHAcp6Q/iximWMRrzhrXwGtQeoXnsIIPN1UTIsKUiEawW2UBdMG2hotCexP9c0qh+MBOR0QEsUhnhCZfh3rDmRLz9wtHvHpFoeEAqtiVflDEL6owkiu9mmeld7rSKyoqRlx33XVPKKXSoz6ZxWKxxBG7b4o9b7755tgNGzY8n5+ff392drb78MMP77z//vvziChYWlo6fv/+/dOvu+46ysrKGoNQkzwHob+TgB5PcjLCer1xsAhb4wWFZs4vg15HlfNIrCpJxzzhAAAgAElEQVT+ebQxVVmI7Ob3753i3iBMJJRSGez6XQhgmBBiPwu6eyXwpNCoML+w1T92wHQymwlcnetFpRT+53/+Z0JTU1PqLbfcUpuVlZXQ62QAUEqN43j3uQAghNjDwoS4jB2KBIXGJEUSF5v1G7N24J8NP+DR/ixMmMEC5EuhayS7eCxWQu+hWbCTZVwDU1xsusm92kEkcXGsjymFRV0LoEe2NQshdjqOszNewhmK7kovJaIj5n0/85nP3Lpr165fENFr8TgWi8Uy+LANdIvFYomC0LGBvwdQAOBV6Fl+SwE8A6AKwDwiCgohfg7gK9AFo6P82DToscOjADwLoA7AKgB3ILSZi8QSIoqodhZC3A9gQ4QfEYCJRNRvm6XeIoRIBTBDSjlr8uTJnzl37tznGhsbJ+Tm5uLUqVOCiDBnzhz8+c9/NmeLeUWHc0Q0k4jmQ88A28lugIRz1CYyvMnKVEpNVUpdC72pEdDNqDTjrm0wCj0IFyoMaTd/JLiIsZSFCS6/P/MjbRKpq4vWLPiYxQZ/fKNZdEv4wk5vYWHCNI53nw1dxNjpOE5BrDbdUa5BLrpGaJ5D14JPvUiiRAUuCs1yXXchdIzmR+z+KYyn+6cbx2XGB4a5I9DVTe6PDuxz7H0fjtsrCs2HnqHYzOdzZyxjXn2vCWhX+mXQf+9BIVd6VWlp6YiPf/zjj7muO/wCT2WxWCxxwe6bYovQHY1cALOys7OvSU9P/0ogEJg3YsQIp6OjQzY36/Hcv/71r7FixYozCEWtNwAIAMgw1vm1juNsklIeGoi/m8kK/+0d4bpuLhHN4TXpSOg9k0BoTJXnoo00pmpIu/kjQUSpruteww7/bADVjuNsj/T+9K7BeRK4zFFhXYQKUsp6JIiwNV6wYHYex7uPAlDFjfTSWL33SDuZwxrkEYStQehr0EVcnEwmBx4nuIBCY5wqeA8as/PZG3zXwC8uNt3kZuy9KS4ekNEPXCNZwEKPdCFEmZRyh5TycLz20kQ0zBAgjyCiVsOV3nzLLbfctnv37p9TlFQci8Vi8WMb6BaLxXIehBArAHwLwFxo5flR6OLQ94joNN9nA4D7oAtB1cZjxwL4AYDboBffuwCsB1Ae4aX+CN0sm0FRvpi5EPTvAL4K4JDvx0VENODq3fMhhLgK+hx4m6xWAGUZGRkNV1xxxRWO42RPmjTp9MqVK7d96lOf2hlto0t6BtgipdRS6EV4ieM4m+PVuEhGDEV+JFeEOeMwCMA7b16so+euODKYiw3xgt0Vi9hdkQ7giNCjB1IQ2ZnizWT2F3wCQoiPBuSXSCC4iLGQhTPpQohDUsq87rw/jaJbWIM8QrHBnIvd2SgfjGIRpVQ2FzGuhXb/HOaZf3GLy6eQm7zTEUHnjw8MQBd8ehUf2J9woscCIroa+vNe7jjODi6yxasolM7uiplgV3plZeXJ5cuXLwkGgyMv9HiLxWKJF3bfFDuEEL8HcLtx03HHcSqmTJmSM2PGjEuIiJYuXXr4a1/72ruZmZnNkZ6DQnOor4ceQ1InpdzMI4dsIZCh8KjpsL0TwtOfzkK/7zL431khRCELZhN2rZKosMDzMtd1l0ILEhuFEOXQ5znLcNH6r0GkEUlDPlGBiCTPol4CYAqAM5yMVtyd/QyFx977xcWmQPOCKWiDAUMwuwjAZACNnDy1U8QpLp9CbvJI4mK/mzyauDgeh9ZnWDhzFbvSJ0GPH9jJ/87G6TUB7Uq/DHosFojo1N/93d/lFhYW/otS6qfxeF2LxTL4sA10i8ViGWCEEDcA+ADAPxHRd85zP68QtJCIdvbT4cUMIcRI6IJZKYDDAI4TkQJ0tPv69esX/ulPf/puSUnJx5YvX1771FNPbVmyZMnhaM9HRGmu685XSi0HMELo2LLNrDQf9FD4TOzO2MYoivxorogmb6PLm5p5HFM4WghRKqXc4jjO0QH49ZIG6hr3bRbdTEd/EMApAEcMJXjYNbBEh9+fc7kolAs9oy7PcZy9/PNso9hgFnzMotsZRHaTD7lrwOdzDs/8mwCggYtCxb1xifjc5GHuCIQXfDw3ub/ok9TXgM/nlVwUmgygyXClxzQlpb29XRQVFWUWFRXlHj16dObJkyenlZeXZx08eHCYUkpe+BksFosleRlC+6ZboN3OhwGUE1FnkzwvLy/jBz/4wdN5eXmPABj24IMP7n7kkUc2p6enR22Sua471XXdG6BHDgW4kb5nqDQdSY+pyjLWieaYKjN5yEx/ajDW6w2moM913SlKqRXcmGmSUm7jxlrSz02OFxQ+G96fwDXad/ezAI5KKU9GuwaW6LiuO0kptYSIrgQQFDqOvEAIESCiUb7z7wlcMxFK/PCS6DxhqzcXe0heA6XURBbIXwUdELJPSlkopTzWm4a14Sb3i4tNkX07IouLk/4a8Pmcz670FK477YiHoLuqqmp4QUFBzoEDBybW1dXNqqmpGbtjx45RbW1tv3Bd92uxfC2LxTJ4sQ10i8ViGWCEEP8J4G74nBgR7pfUhaDu8tJLL8175513XiwuLv7k4sWLTzz55JNbr7vuulLoeLwusDp4nlJqBYBRQoh93Eg/2b9HHh/MBi26uiLMebeeGtws9gRED6OmWb0+h4UJY6Fj9bZwzFZMf7dkgRX5WVEKPqbj04z77hxBQERjufE7FbpRuZ3neif15rc/4VSFUVxsuIz0fLpM6O8F843ZinBnivffQR1731vYnTZVKbWQi2yKxUgFUsoTEe4/nL+POh0RF3CT1yNUdEtoN3msUEpNMIpCqezyL+Lv0G5vvE6ePJm2ffv23D179uQePnw4p6KiYsyJEydyGhsbxzuO05ySknLYdd29HR0d+yZNmlTjOE7T0aNH34nmxrRYLJbBgN03hdi4ceOI9evXr9m+ffuqYDCY8cADD+xdtWrVpuHDh0dNMOLG2vXc+D3DYtldIgHmEPcVX4PWFLdmQ68ZPbzGlJn+5I2p6pGIkJNolvHf/HYhREFKSkp+vByqiQ47PDN85z9a+lOXyHUAHUqpa0jHZ0shxG6Odz81ML9RcuKlKriuO5mbvlOgk+bMfZMnsveugZfANehj73sLEQ13XfdaFsxmAaiVUhayGCnou6/pJveLi/2O/mji4v775QYA0smSc/h8jod2+RdJKYullOe6+zzt7e1i586dmTt37sw9ePBgbnl5eWZ1dXVmIBAY39HRMTItLa1KCHGwra1tN4BDc+fOVSUlJe8TUU3cfjmLxTKosA10i8ViGUCEEKMB1ALYTES3XOC+3iy/k9CNzUYAHwJ4gYj2xflQ+52XXnppzrvvvvvdoqKiW6699tr6J554YttNN920H9Eb6Q7PU1sBIFPoqOdNjuMk/MKYXRHZFNkVYTZomxHdFRHT5iCF5iZfB73prpNSbnUcZ+9gdKtQKPbeX3TzK/I7i25m0ac7cd/sVllKeiZXqxCiMCUlpUAIETF6cyhiCEY8Rb7ZoE3luykADdDulOHQ34eOEOIgv0ePD8jBJzlENJLFSAugnUD1Qoha0jG32ejqJvdEO/6iT8LGB/YnpFNSvKLQhJqamrM//elP6++7776/zp49+wQAKKVQUlIyeseOHbkHDx7MLSsry6qurs48derU2NbW1sy0tLRjUsqDHR0de13XPQDgIIBDRDQkklYsFovFxO6bIrN79+5hL7/88uqtW7eubm1tzfzKV76y7/HHH9+UkZERdX2plBrvuu71RHQFgLO8ftqZ6GJDbtCOitKgNd2bLiI0aHnPdC7W6xSl1GjXdZeSHjkkhBA7ufF7JqYvlCBQeOy9f99kCry99Cd/5Pp5Z8MT0TBOmlsMLZAvk1Juk1JW2jWmxhCMmHHf3jUYZdy1GUAAuoaSzT87zYLuXUNB3BpruE5yiVJqERFdAu3YPwo9hiDjAm5yv7g4ob9z+wMWdE9WSi1gwYd8/fXX6yZOnLhn5cqV21NTUwkA6urq0rZv357D4uLcysrK0bW1tTmNjY0ThBAtqampZUqpfe3t7Xuh90wHAVQS0ZA/xxaLpW/YBrrFYrEMIEKIrwN4DcBdRPTmBe57M4DlAPKgG0dzADwLvThfSkR743y4A8Krr756+W9/+9sXd+zYcftVV13V8MQTT2y/+eab9yJ6I91zUF8HvYEscxxn00BHkVMo4jiSK2KMcVcvMq3TwdzdBm2cjhtKqWkcUzgLWhm8jaOek27DbThoI0Wu+2PvAwjNF/P+9bnoppTKcl138VB1VxiK/JwIBR8zRvMcws+/97/DUhVYvX4tz6jLAnCMZ/7tH4xij1hBofjATkdEBIcQoD8PASFEmRDiuDGCIOk+/wMBEaGpqWnqb37zm0++8MILU9vb2zFx4sRzjY2Nzc3NzWMAuGlpaeVEtL+trW0PQgWfciLq9+98i8ViSVTsvun8FBcXp//whz98ZNu2bU+dPXt27L333rt/9erVGzMzM5uiPYYd1CuIaA6AZl7jF4kBjCLnGuUIpVR2hPW6KagkAGciNMkv2KCN47EPd113ETd+h3Gqz1YpZV1/H0tfIZ3AlWmu1aOs102Bt/nvdF+bgyyQv5JHjI0HcIIbv/sGQ2pCdyCiEVHExVnQrnJAO/q92kHn3smfqsD7+ulKqcVEdDmANhZ7FMR65NBggveu5nz4zuh1hLvJAT0G4iiLPsxxYf1/4EmI67rDamtrF919993X79+/35kwYYJqaWmpb29vT2ltbR2TlpZWw+LiPYa4+KAVF1sslnhiG+gWi8UygAghCgFMBzCZiHrcjBBCTAewB8DfiGhlrI8vkfjRj3508e9+97sXCwsLP3f55Zc3PvbYY3m33377bujmTheISPCG+3po50mV4zgbpZRV8drAsIs5misiExeIrZNSNiCBI9PYrbKclcEfcZOyUAgRNSpyIPA5+v0FH3OTexaRCz49ir3vw3EOc113ATd+Rwkd9bx9MLgreH058jwFH3/cd6SCT4+ah+wGuNR13cUAZkDPpCzkQvBQjdI04wPDotfR1U3ujw4MEFEax7tfAx1HfkhKWTAY3qOxRimFsrKykfn5+bn79+/PLS8vz66srBxTV1c3trm5OTc1NfVkdnZ25ezZs8du3LhxJBFNAnAEwE8AbCCipCuuWywWS39i903do7CwMG39+vVf27p16z/U19dPvOeeew4+8cQTH+bk5ERtkCmlsrmRfjWANm5SFsRTvOtzMfvHVA0z7uqt1yONqUrIJioRpXKqz1IAY3iNv8VxnKhjBwYCdvSHJXAh3NHvLfbM9XqnsFVK2SB6GHvf2+NUSs1wXXcZgEug1/j5vMaP++vHG+qaRmeKiyPFffvFxT0WjCilMl3XXciC7nShk7zypJTVQ3WNz2KFSOJiv5u8y54JQKNSarZSaiGAidAu/x1sOkioWkki0NjYmJKfn59dUlKSW1pamltRUTH6+PHjOadPnx5PRJSamlp+9dVXt1VXV2eeOHFiOj/s/wJ4HTqFxjazLBZLv2Eb6BaLxTJACCHmACgB8CoRre3D8/wPgGu4GD/oWb9+/UVvv/32PxYWFt45c+bMplWrVhWuXLlyJ87TSFdKXe667vUAJgA46jjOJillWW82h77GYHYEV4TpYo7mimgaCFdErOAN91LecJMQoogd1Gf76xhYrBDJ0Z+DcEe/NxO7s+jDkWkNieKgZXfFVVxk89wV29hdkdAOai745EQp+JjFzzOIXvCJ+XEppcaxy38uALADKC8ZHUDdwXCTdzoioogVzEJPfXc/C6TjyOey2GMsP7bQcZySgUjGGEhaWlpkYWFhVnFxcW5paWlueXl55rFjx7IaGhrGB4PBtLS0tHIAB9hNfgjaGVFKRJ1RukK/6ZcCeBjAlwA8R0Q/HIjfx2KxWJIBu2/qOYFAIPXpp5++f8uWLd84ceLEtLvuuuvgU0899eG4ceOixoorpcZwI/1aAB1CiHye6d2rBhC7mM3GYLaxTjRdzC2IPqYqIdbrvYF0MtpVPGJsLPQ+dIuU8nB/7gWJaFiEBC7/iCSCTuCqR+S9a38d7nnh1ISlvMZ3hRDFvMZP6Lh8n1jBFHjnQgvtPdoQ3qA1r0PMo6hZ7HE1pybkQu9D83hsW0IKVPoCfydFExebYgVz72qOqzrvZ4HFHlNYgHwldK1kL7v8a+P72yUWSilUVFSMyM/Pz923b19uWVlZdlVVVWZdXV3uuXPnxqamptY5jlMaDAb3BIPBTjc5gBqzQS6EyAFwP/S+aSSAi8jGslssln7ENtAtFotlgBBC/CuAxwDMIaL9fXieIVMIMtmwYcPkX/7yl98pKCj48pQpUz569NFHC++8884iKWXEjR5vZma5rnsDgMkAariRfijSJohCs5gjuSLM2W7eHOBIroiEbn72FSIaGQwGFxHRIgBpQkeRb5VSxiRCyydWiBTh6MXWeXMOwwo+UsoAgJZEKfhcCH6PzmR3xcXQMyk9d8WANSkpfD68qcqPJFbwF3zqRQwiHPtw7CPYAbQIeuZfFRfZSpNNxOJzk4e5IxDZneIv+vS5+Emh6MdFpKMfXSFEieM4hYNNnHDs2LFh27dvz927d29ueXl5dkVFxZgTJ07kNjU1jXcc53RKSsph13X3dXR07EOo4FNNRD363hdCZAMIElG/CZAsFosl2bD7pt4TCARSnn322Xs2bdr0zPHjx2feeeedpWvWrPlg4sSJDdEeo5Qa5bruMiJaAEAJIQpTUlK2CyG6zFWn0CxmU1CZDb1ONBuD3hxgT1xsjqlKehfx+WBB9yzXdVcAmArgFDvS98Zqv8iiVrMxaO5dzdShJkSPXE+avatSKoMd1Auh4/IPsAD5+EAeF+n0prAGuXEdPLGCNy7M3LvW8951QNLo+D06Uym1hPRc72YhxA7HcXZIKc/1+wH1AQqNgYgmLvZOsOcm9++ZGmKxdyWikTxmbAH0nvm4lLJgsI0gaG1tlQUFBZ64OMcTFwcCgfHBYDA9LS2tQghxqLW1dTdCe6ZSIurR+4oFyFOJKKGSPCwWy+DHNtAtFotlABBCpAGogV44LuvD88yEdmP8mYi+EKvjSyY2bNgw4Ve/+tW6/Pz8e8ePH9/+yCOP7Lj77rsLLtBIn8mO9OnQm9ZDAD4CN8qhN7gjjYeZs938roghr35ld+p8dlCPEjoCbqvjOMe6+3giylZK5YIb5FHECmcQueDTmGzN0AvBDuqlpGdSukLPp8uPp7uCRSNhDXLjOpjJCn6xQj0XPxM2Jp0dQLOVUksATIGO1SvgWL2EclBT+KxD7zr43eReEbreuxbsJo+LOyUSSqlRSqn5Sqn50C6yI+xKP5AsBdhgMCiKi4vHFBUV5R48eDC3vLw8s7q6Oqu+vn5ce3t7RmpqarWU8mB7e/sepZTnjDhERAntcrJYLJbBhN03xYZAIOA899xzX9y0adO3qqqqLl25cmXZ2rVrP5w6deqpaI9hsexSblJKIUQpgOPQf/fNqG9vfeKJWv0JXA1CiHPJImqNJ67rTlNKrSCiWQAaOS5/Z3ec9ixq9YsVvLW638UcLYFrwObbxwN2UF/Da/xsANWcjHYoXvtDFo1Emg+fi/BkhXMw9kvJIlZQSuVwktc1ABwhxF4WddcM9LGZ+NzkfnFxWBIa75f8jfJ++U4iIskCmkUAZkIL/HeyOCFpZs8fP348PS8vL3fv3r25ZWVlORUVFaNPnDiRe/bs2fGO45xlcfHeCOLiQSMWsFgsQxPbQLdYLJYBQAjxJQC/BPA1Ivr3CD//OYCvAJhJREf5tr8A+ADAXujN2FwAT0M3epf3xY0xGHjzzTfH/uIXv3guLy/vq1lZWe7DDz9c/NWvfnV7a2srdu/ePfXQoUPTbrvttuDo0aMzDQez6Z4l6AbtMVZ/m66IhGqwJSqko8jnKqWWQxdyjnBMYRkAzz0bqeAzyngaM8LRLDgMmIt5IGF3xSIuXKYLIfazu6JXBQzDxRzJTW6KRkx3illsaEzkgk93cF13ErsrrgQQFELs4li9QH8dA1+HbOM6dLoj0DU+0B8dGBM3eawwxAkLoUVJ56SURfyvaaCPDwDq6+tT8/Lycnbv3p17+PDh3IqKijE1NTXZjY2NE4QQbampqYeVUvva29v3IlTwqaRezNi1WCwWS2yx+6bYEggE5Lp16z67adOm58vKyq684447yteuXbtx2rRpJ6qrq3N27949ffTo0RkrVqxIQXjyU6rxNG0AaoUQJ31r9h7PYh6qsFh2OYtlW7lBWQAt6vbElJEi170Erkii1sBAupgHEnZQX+a67lIA0wA0cBT5ru6IEyI8HxByMfvd5NkIiUaC4LU6uoqLk7qGQETD2EG9CFqgcYzPab+JZSk8kS6SuNh7o3uiEf+eKaEMDyxOWMjihDQhRKmUskBKWZkI353BYFCUlJSM3rFjhycuzjpy5Egmi4tHn0dcfHqgj91isVjihW2gWywWywAghPgzgCUAJpExF9X4+QYA90EXgqr5tn8B8Cno2LfhAOoAvAfguwC+DT0XKBpLiKjgPMczFsArAG6FjngrgZ7J+n7Pf7v+RwghoWPZLx0zZsw1w4YN+3JDQ8PcESNGpDQ3N4tgUO+Zfv/737sLFiwwowMDQogGAGmu6y4EcCm0M3Wz4zi7B1O0Vn9AerbbKN7gXkk64jkD2o0iEdrgRio0BKSUDaKXMxYHO4a7Yil0saDacZxtkaLIjUJDJDe5fyZ2Z4Mceq61dz0GlTslEkb04wIAI4QQh6WU+VLK8lgVHdlNHuaIiHAdvPhAf9Gn39zksYKLwQuJ6Gpox8pBLgodiXchVymF/fv3jyosLMzdv39/Z8Hn1KlTY1taWrLS0tJqpJSHOjo69rquux+h+eSnyG6ILBaLJWGJ8b4pA8DK87zcoN4zAYAQYhSAWY7jXDZ+/Phbz5079+mmpqasUaNGibNn9TSRW2+9FT/72c/OggXFhpO5SSk1i4iWQAs7SxzH2SyltM2THkJEqUSU7bruNNLzvCfzjxRCTXIgNC7M/+9MIjTcEhHXdScrpZYR0WwArUKIQh431CUymnT0vSlqNcXFYS5mRBYXNw3262CIExYDuAh6zFghjxmLyd6d3eTRxMXedSBoN3kAXaPXkyrhgnSC3xwWJ4yDTokodBynRPTDSIuGhobU7du3e+LinIqKitE1NTU5Z86cmSCE6EhNTT1MRPva2tpMN3kFEQ36GoHFYrH4sQ10i8ViGQQIIWYAGBvhR3+EdvTOiNYgEDoWsQjAaADfBHAKwCoAtwG4iYg2x+WgY4gQogTaWQLo5mxlSkpKxcyZMydNmjTp0mAwSDfccMPBhx9++E8jRoyIuuhXSk1wXfd63mw38my6XcnWxIo3RDRMKZVtbGxNN7nnTiEAp6HffxnQqvVzQoh8jiK3zs5eYBQwlkEXhRsBlAshmgBkGQUfL/reLDREKvgMyO+RSBBRiuu6VymlFgOYAC0kyOcCRnfiND03ea5Z9IG+Dn43eaRG+aC7DkSUzoKPhdDnoY6LQrv7Ks5oampy8vPzs3ft2pV7+PDh3PLy8jHHjx/PbmhomEBESE1NLQdwoK2tbQ9CBZ/DRDSo56taLBaL5cLYPZO4D8B/GjfVAyjNzMxsvuaaa+a3tLSMmTlzZt3999///sKFC8uiPQ83fxYopZZBixD3cCO9Pt6/QzJB4VHf/gSu0cZdP4LeNzn8M4edqRsdx6kdgEMfFCilMl3XXUJE86CFq5VCiBoAw4zrYEbftyJ8v5SQLuaBRCk1nuPd5wIgIcRu3tvXXeixUUTeprjYdJNH2jMNuutAerzgNKXUIq5BuXxOC6WUJ/vy3EopHDx4MMMUF1dVVWXW1dWNbWlpyUlNTa11HMcUF3v7pjorLrZYLJYQtoFusVgsgxQhxA3Q0YX/RETfOc/9HgXwbwCWeo4LIYQD7ahoIqKl/XC4fUII8VloF+dhAFVm7G5eXl7GK6+88vT27dsfATDswQcf3PPII49sSk9PjxqpppQa67rudUR0FXQc8TZWWA+Zpq9PBe4v+JhR3+ZsN7PgcMZ08LM4YQURXQGghRuUhf2hsE5mqOusw1zjOozx3f0sgGop5UlfocEmKXQDLmBMV0ot5vSENqHn0xUIIRoRHuNoFnyyESr4eG5yf9Fn0BV8ugOf05lKqYVEdBmAdiFESVtbW1FGRsZ5C22VlZXD8/Lycvft25dbVlaWU1lZOfrkyZNjz507Ny41NbXecZzSYDC4JxgMmgWf47bgY7FYLJaeMMT2TDMALAdQCi0u63SOBwIB8corr9z4t7/97R/37du3+BOf+MTRNWvWbJ47d25VtOdjEeI8pdQKAKOEEPu4kd6nxk8ywQlcGf79EoUipv1R32Hz4Tn96SPj+dL4nC6DPqeHWNR9rN9/uSSDiNKNqG9/9L05gqADwAkAR3k2fIDFHy2DTdQaL4hohOu681ksOwpApeM4eVLKw9Cj26KJi/1uci+Rztw/JZWbPFYopTKUUvOVUvOhz2k1gML29vZDGRkZUetQzc3NTn5+ftauXbtyS0tLc8vLyzOPHTuWdfr06QlKKZmamloBYH8EcbFN/rNYLJZuYBvoFovFMkgRQvwngLthxBlGud9fAEzhxqZ5+zMAXuKfJb3yfePGjSNee+21p7Zt2/ZYMBjMeOCBB/auWrVq0/Dhw6NuHHhG1QpWWLdyI72wrw7KRIGbs6MjNMlzoZuz/qZgZ9Gnt1HfSqks13WXEdG10ArrHbzZTohZyQMFu/qjFXxS+G4uOEoT4dejQyl1jeeuYNX6dinlqYH5bZIXFo5kcSN9DoAp0G4gF+Fxml7Bx98oH5IFn+6glBrjuu58Ipp/zz33jGhubv7os5/9bNGll166a8+ePTmHDh3KLS8vzzx69GhmIBCYEAwGh6WlpVUCONDe3r6HiLyCTykRDQkMf0cAACAASURBVOnvC4vFYrHEDrtnCicQCIhXX311+V//+td/2r1794obbrjh+Jo1azbPnz+/ItpjiMjh5JkVADK56bvJcZyafjz0uEJE6USUrZTqHI1kiFrTvLshPOq71/Ph+ZzO4XOaA+CI4zhbpJRlQ3mtya7+rCji4gzjrp7Iu3OtDuA0EU3mkVjjAZzk/f0+KzjuGZ6b3HXdsUQ0h4guhT7/CrqG4HeTRxIX23MeASKSSqnLXdddtGnTpumPP/443X777dWf/vSn3zt58iRYXJxdWVk55uTJk7nnzp0bl5KSEnAc57AhLvZGVR0jon6ZV2+xWCyDFdtAt1gslkGIEGI0gFoAm4nolgvctwbAJiK6y3f7ZwD8AcDNRPS3uB1sP7N79+5hL7/88upt27atbmlpyfrKV76yd/Xq1ZsyMjK6zFT04Pi3Fdz0bZdS5jmOk58s7mkiGh6hOes5Z73mrEKoOesv+MS8KcjzpxcT0UIAKULPUNwmpQzE9IUSCK85G6Hgkws9R9OjCUaD3Ofqj7pwI6JhHKe5CNqxclhKuV1KWTmUC21+eO3ruck7r4HhFDILPg3Qn40s6GtUL6XczvHutujTTWpra9Pz8vJy9uzZk1tVVZXb1tY2u7q6OufAgQMCgHIcp0YIsSkYDBYhVPCpIiJ7ji0Wi8USN+yeKTqBQECsX79+4Z/+9KfvlpSUfGz58uW1Tz311JYlS5YcjvYYIpLc9L0eep9R5jjOJsdxjvbfkfeeCGt1s0luNmebETmB63Ss14cUGt+0AnpO+kkp5VZu+g7KxhiFor4jiYtNV38HQq7+zr0TC72jps1xQtIMnpN+CYAmTkcrSpb9fX9hpNJd0E0OLVoYzj9zhRC7+b3aMDBHn3y0t7eLoqKizJ07d+YeOnQo9+zZs9PPnDlz0c6dO9NbWlogpTznOM7ujo6ODxFykx8iorMDe+QWi8UyeLENdIvFYhmECCG+DuA1AHcR0ZsXuG8bgJ8T0aO+25cA2Abgy0T0q7gd7ABRXFyc/i//8i9f37p165rGxsax99133/7Vq1dvzMzMjOpsVEqNZvf0fOhNYUFKSkqeEKKlHw89IkSUStEj1805zGcRueDTOBBFGNKzkhcopZYAyBBC7OeNdlI6VnwxjmZz1j/brR2h61CPPrj6IxyDwzO9PXfFCW767h2shbZIGEXQXLPoAy1Y8McHeoU30x3RKRzh4uVMpdQSLrQ1c3rCDinluQH5BROMYDAodu/ePbqoqCjn4MGDuWVlZVlHjhzJrK+vH9fW1jYmNTX1qJTyYHt7+16l1AEAB8eNGzeqrq7ui9DOPwng1wD+nowxHBaLxWKxxAu7Z+oeL7300rx33nnnxeLi4k8uXrz4xJNPPrn1uuuuK4VeR3WBiITruldyI30sgCrHcTZKKasGWtTJCVyj/Olb/N9MRFiro2vketTmbByPG0qpi7iRfjGAM+yeLhZJOiaIiFJ8e1fTTT7MuKvn6q/37V+beuLqjwSPbltKOnFOCT2+KU9KeaYvz5tMUPhscr+42PxMmDPizWsR5iZXSo1yXXch10xGCCFKpZT5UsqKgf78Jwp1dXVp27dvz9mzZ0/u4cOHcyoqKsbU1tbmnD17doKUsjklJaVMKbWvvb19L4CDM2fOrKuoqFgC4BEAlwPYB2A1Eb0/oL+IxWKxDAFsA91isVgGIUKIQgDTAUy+UCNiKBeDACAQCKStXbv2wS1btjxdX18/8e677z7w5JNPbszJyWmM9hh2Ty9l9zS4kbYt3o00jqwb4y80QBcZRht3bUW4g9n71yASdI476RmKV/O8v2zoOWpbEnWjzYKFHKVUpyPCKPike3dD+Gw3f8En3scIpdRM13WXAvDcFXmO4+wcLO6KCA4VL1YzkpvcHx3ofSZ6VHTk0Q6LiegaAI4QYi+7VpJS9NFTTp8+nZKfn59TUlKSW1pamlNRUTGmpqYm5/Tp0xOEEMHU1NSwgg//KyeiqMIQIUQ2gAcAzCGir/bPb2KxWCyWoY7dM/WMl156ac6777773aKiolvmzZt36sknn9x+44037sd5GukcRXw9gAkAjjqOs6k/YshJj0fyBJT+BC5vHrYCcDpKAlfc1+q9RSk1wXXd5UR0JYCPeH1fmIjrewofGWZeD29kmIe5fzX3sT1eq/cG3t8v5P39MCHEARYoHI/3a/cXPje5X1xs7l+9z0TnnolnxDf35DPB+/s5SqnF0KLuU7xn2p2oNYlYopTCnj17RhcWFuYeOHAgt7y8POvIkSOZp06dGtva2pqZlpZ2XEp5sKOjY4/rugcQcpPXR3tOoS/AjQAeA/DPRLSjn34di8ViGbLYBrrFYrEMMoQQcwCUAHiViNZ24/5DKo4wGoFAIPUb3/jGV7Zs2fLN2traaXfdddehp5566oNx48ZFVZ8T0YhgMLiEiBZBN9J2Oo6zVUrZ6wgtnwI8UsHHi6xzEcUVAaAlUQs+F4ILbbPZXTERQC030g/01WHQm2MxBAtmkcEvWGhB1wa5V/BJiAhqpdQ4Fn3MgU5P2Ok4Tn6yuCuMgk+OWfRBZDe5J1gwG+UxH0NAOjL/Wo7MzwRwlItCB5Ld6a+UwuHDh0fm5+fn7t+/P7esrCz7yJEjY+rq6sY2NzfnpqamnnAcp5QLPvsRapSfJLu5sFgsFksSYPdMvefVV1+9/Le//e2LO3bsuP2qq65qeOKJJ7bffPPNexG9kQ6l1CzXdW+AjiGv4Ub6ob6sz9jBHC2Byz8eKdJa/Uwyr9mUUlmcjnYt9Pp+B7unoyaqxQsiSvftX01xsX9kWJi4OJH2r0SUyqLupdB776MslD/U33vR3kBd09BMcbHfTR5NXBzrMQRQSl2klFpMRJcBaBNCFDmOUyiljGpaSBYaGxtT8vLysktKSnIPHz6cW15ePqampib79OnTE4jITUtLKyei/W1tbXvATXIAZUTU70kWFovFYuk5toFusVgsgwwhxL9CK1LnENH+btz/zwCmsILdvP0ZAC9BOzJOxOVgE5BAIJDyzDPP3L1ly5Znjh07dvEXvvCF0jVr1nw4adKkqLO5uZG2iGPI04UQxdz0PV/zPc3nYDYLPunGXb3IOn/B52wybOJ7i+GeXgFgBoAGdgGUxNqFEMGhYhZ8HL6biygFHyHER7E8nnjC7opFRLQA2l2xn+PdB9xdEUE8kovI8feem9xf9BkQwQKFZlMuBnARgLNSykKeo5jQ742WlhZZUFCQXVxcnFtaWppbUVEx+ujRo9kNDQ0TlFIpqamp5QAOcMHHm01eSkQDPrbCYrFYLJa+YPdMfefHP/7xJW+//fY/FRYWfu7yyy9vfOyxx/Juv/323dCN0i4Y6/vroZ3/J7mRHlUo6xO0+hO4TAezmTYUQGg8UoPo43ikRIeIRrKoeyGAVCFECYu6o+5fe/k6kkIz4v1u8pHGXZNesGCs75cCmAa9F81zHGdXIrinfeIRv7jY7yYPExfz+6JHbvJYoZTK5L3oPABp7PTPl1JWJ4KAIhpKKZSVlXWKi8vLy7OqqqrGnDx5cmxzc/PY1NTUk47jlAaDwb3BYNAUF9dacbHFYrEkN7aBbrFYLIMIIUQagBroBseybj7Gm/23hIgK+TYH2pHRSETL43W8iUwgEHCef/75Ozdu3PjtqqqqS1euXFm2du3ajVOnTq2L9hgiSnNddyEr1kcIIXZLKfcAcCIUfDKMhzaja2xgQAhxuj8i6xId13UnKaWWE9EVAM5x8WKH6MH8QQrNw47kJjcdKuaMeLNB2ziYBAvsrriG36tZAKrZXVEa79/TFx8Y5o5AZDd553XoTXxgf6KUGs/x7nMBkBBiNzv9o35v9AfV1dXD8vPzc/fu3ZtbVlaWXVFRMebkyZO5TU1N4x3HaUhJSSlzXXdvR0fHPoQKPkeJKCmKnBaLxWKx9AS7Z4ot69evv+jtt9/+x8LCwjtnzpzZtGrVqsKVK1fuRJRGOgC4rjudG+kzAdRLKfMBnAKQHSGByy9ojZTAlbDrw/6CiNJd153P6/sMbk5u6cmYoShpaKag1UtD64AxlxxasOCt2QeVYMF13clKqaW8F21lp39BP4xv89zkkcTFkdzkfnHx6URJQ/PDdZOrOd49B0AtJ3ntHchjbmlpkTt27MjyxMXl5eVjjh49mtXQ0DAhGAympaWlVSAkLvb2TKVE1DxQx2yxWCyW+GIb6BaLxTKIEEJ8CcAvAXyNiP49ws9/DuArAGYS0VG+LQ1AEYBRAJ4FUAdgFYBbAdxERFv66fATkkAgINetW/fZjRs3Pl9eXn7lHXfcUb527dqNM2bMOAEAruviwIEDEy+77LJhQgizST4J4Y3ZILgRCF/BRyTgvLpEhGdPLyOiqwEEhRCF3Jw8B0QtMuRQVwdzO8ILPp6TPJAIjoL+xHBXLAMwFdpdsZ2d/r0+F77iW65RfIsUH+iPDhwwN3msIKIRXMBcCP3dWsmRmofjJVBob28Xu3btGlNUVJR78ODB3PLy8szq6uqsQCAwvr29fWRaWtoRIcTBtra23UTkFXwOEVHSRydaLBaLxdIT7J4pPmzYsGHyL3/5y+8UFBR8ecqUKR89+uijhXfeeWeRlNIFgPr6+hGtra25EyZMGIVQAtdE6AaaNJ6qEZEdzINK0BovSM+enquUWg4tQKjkdLQKT2RARKk+B7MpLjYdzGeM/au5Vk/YGfHxgt3Ti9k97bBQdruU8lRfntdwk0cSF0dyk/vFxQkRf98beC96seu6SwBcDKBZSrmD/8VNoHDs2LFheXl5OSwuzqmoqBhz4sQJT1x8JiUl5XAEcXG1FRdbLBbL0MM20C0Wi2UQwdGCSwBMiqSCFUJsAHAfdDGo2rh9LIAfALgNuum7C8BzRPSBEGIFdJFoKbQ79BiA/yCil85zHPcD2BDhRwRgIhENqBuzN0ycODHr4osvfuDIkSOP1dbWTr/iiivag8Fg6pEjR5yWlhYUFxdj/Pjx3sY2ACAAoAHaRXEFgNFCiIMcU1g7oL9MksPz/m7gCE0JHXMfhG7MpvHdzGthFt7qRRzmYQ8G2F2xjIhmQ7srClNSUgqEEFEV9T43eZg7Al0LPt61qGfxSEK7yWMBEUnXda/g8Q6TAZxmd8WuniQomNTV1aXl5eXl7NmzJ/fw4cM5FRUVY2pra3MaGxvHCyFaU1NTDyul9rW3t+9FaM5eJRENKXGIxWKxWCzRiMeeiX8+5PdNQoiUpUuXLjhz5szzlZWVN40cORIzZsxwjh075pw4cUI8/PDDWLduHQC0wBAVA3CVUpdAO9IbpZRbHccptmlcvUcpJZVS84x53i3Q8erDEB5//xG6NsjjMg97MEB6fNt8dk+PAlDGjfSKaPsan9Dbv2fKNO5qXgvzeiSsmzxWKKVyOd79GgBSCLGPBci9qp0Eg0FRXFzsFxdn1tfX///s3Xd8XOWV+P/PuXdGtrGxZGlcKaabBNNjcCeEBAhLdhMCWSAJZYlZAsFgDARSSBZCFkII2QQ2YTcJSX6k8s2GZbMsxQZsbKtYbpIsq0uWi1w0ci+S5t7z++PeweNBkpu6zvv18sswunfmGcmYOec55zyjWlpahkej0bWO45S1tLQU+76/hiBmKlfVbZ36xowxxvRptoFujDGmXSJyM/Bb4I8EXRq7CSqDx6nq9zq471bgV8BtBIFIqmWq2ieCPxGJAO8AE4BRycdd142PGjVqyPjx44eMGTNm78UXX1zxhS98IT87O3urtHGuW7iJdr7v+zOAESJS6TjOQtd113ffu+lbNDjvMCt19H1ynCMwPOXSVoKxjg6w2XGclY7jVA2EJENXCbsrJofdFY6IFInIKhEhtSOinW7y9NGBlnwLhQUKk8OCmkRTU1NRXV1d6eTJk2vTr/V9n5KSkuMLCwtjpaWlsZqamhFr167N2rJly8h9+/aNyMjI2Og4Tllra2ux53nJrogyoFHtw70xxhjT7Sxukq8C9xG850j48L7BgwfvmThxYvbw4cP9CRMmbL766qsLpk6dWiki+9p6Ht/3R3qeNzMslN0TbqQvG2hToo6Eqg72ff+geCn8nJ7NgZ+FTzACP0pQKFsuIitd193c3s/CdExVXc/zzgkLFMYQxKL5ItJAUESfvlGeWuidPIrgoOJiEdnbM++m9wgLFC4MCxQygfolS5aUX3rppflDhgz50N+HjY2N0fTi4o0bN+Zs3759tIi0pBQXp3aT11hxsTHGmMNhG+jGGGPaJCLjCJI4v1bVe4/w3mQiaJKqLu+K9XUXEXkB2AxUEHw/qlR1Vzwelx/+8Icfnzdv3uMlJSWXXnHFFevmzp37/nnnnVfX3nOFG+nn+L4/kyCIrnFdd6Hrumu75c30Qqo6pIOET+p5h3FJ6yYPkwz7wzGFF4RjCrOA6nBMYV1/7m7ubCnjA3NUNeb7/miCse6pBQup4wM/2Ci3sycPn+/7x3ueN+mFF1649Mknn8yYOnXqvhNPPHFlS0vLvpqamswNGzZkb9u2bYyqajQarQFK087Zq1JVO/bBGGOM6SUsbgIR+Qfg4wQxU/LXBlX1//KXv2T/+te//kZeXt4dw4YNY9asWSvvuOOOJZFIpN0NrPDopunh0U37wiOGCqSfna99uFTVUdURaWO+k8XFQ1Mu3UXb3eTbRUQ9zxvn+/60sKBzT/h9LTzayUgDUdhNfnxKzBQDTgRGc6BgAYJu8raKi63Q+zCoquP7/oQdO3ZMvvDCC08ePny4Tps2rS4SiVQ2NDQcX1dXl7l169ZRe/fuzQ6Li8tbW1tLPM8r5UDctNWKi40xxhwL20A3xhjTJhH5DvAYcEry7L8juLdfJIIORzwel+eee27a22+//XhRUdH0yy67bMMDDzzw/sUXX1zT3j3hWV8f8TxvJkGgXe+67oKOxr71ZSljvtO7yWPAkJRLd/LhceuHfd5hyrjs6QTf1w2u6y52HKfMzksMaNvnxCd/Hm11kzcSHEVwnKqeSZCk2xQm20ramrhgPqy6uvq4/Pz82OrVq2NVVVXZdXV1mdu3bx95+umnj9q0aRP19fUCbAPeAF4GVgEbLeFjjDHG9H4WNx2e+fPnD3/++ecfWbJkyZ0ZGRnRWbNmrZo1a9aijIyMdjfFw8lI01X1QqDFcZw813XzRaTfFROmfE5vq7h4BAc+p7eQckY8EHccJ/nvh1VgEBYoTA0LFBIiUhiOy+6yc6f7mrC4ODV+TY1hk93kPinFxcB+VT2RYBKDLyLLw+/r9p55F33Lnj173Ly8vOyVK1fGKioqYtXV1ZkbNmzIzsrKGjdixIiMwsJC8X3fA1YAvwfmERQX2xQFY4wxXcI20I0xxrRJROYB5wNfBJ4GJhJspP0X8LCq7urg3uRZfpuBkcAO4D3gMVVd3bUr7xnxeFyef/75SW+88cYTK1euvHzatGkNc+bMWTR58uQqgq7dD1FVfN8/y/O8y4BxBBu+Cx3HqehrG+nplfhpCZ/UjdkWUirxOTjh0ylj1MLv6xme500Hxoevsdh13aKBUu2f0k2e2qUSI9gATz+bvJGUn0l4Nvne9D+D4ff1NM/zpgBnALvC87yX9cck5pHav3+/U1hYmLV8+fJYeXl5rKamJmvdunVZ8Xh8TCKRGJSRkVErImXNzc1Fqpo8m7yCYMTrZQRjT/8B2A5coaoreu7dGGOMMeZwWdx0ZPLy8ob94Ac/eCgvL++rwOA77rij+Ktf/erCQYMGtdsF7fv+8HDD92LAE5GCSCSS1xdHXqtqVFVzfN//IF5K6SZP/Zy+PW0CV/L3XZ0VK4aTkSar6scANxzrvsRxnKZOeYFeLiWGbStmSj+bPP1c8uTZ5B8qKPZ9f5jneZNUdRIwWETWOI6zxHXdDd301nq12traIXl5ebHVq1fHqqurs2trazM3b94c27Vr1+hoNNroum5FIpEoTiQSpQQxUxmwgaB44U7gbmAs8IKqfq3n3okxxpj+zjbQjTHGtElE1hBsPrYA3wfygEnA4wTn8c3s4N6rgGnhPTuBc4FHCSrnp6hqSdeuvmc9+eSTF73++utPLF++/FOXXHLJ5jlz5iyaMWNGBR1vpJ8edqSfDGwKN9J7Xee0qmZ0kPBJPdctdcx3asJnd3cWB3ied2I4pvBsgg3f3HDDt8+Pf0zrJv+gIyKlaCFpHynj1lN+Hkc9PtD3/VGe501R1XMJkpgrBkp3xYYNGwaFCZ+cysrKnNra2syGhoacnTt3jnFdd2ckEmnrnL21ehhnmIrIqcBXgMdV1UZpGmOMMX2AxU1HZ8GCBce98MILc5YsWfK1RCIx7Pbbby+55557Fg4ZMqTdbtJwY3JKuDFJ2Dm9pLd1TquqqGpmanFxSvdy6vFI6WO+k7FTU3cW/mpw7vTHfN+fDBwnIqXhJK+G7lpDV0p2k4fj1tM3ytvqJj+ouPhoz4lX1ajneeeH56RnA+vCP6/lvS3O72wtLS2ybNmyrOXLl8fKyspi1dXVWevWrRsRj8dHt7a2DgmLi8tTiovLgPKOCo6SRCQDuB7Yqap/6+r3YowxZuCyDXRjjDFtEpFygi7TR1T1mZTHZwPPAZ9U1XeP4PnGA8XAPFW9rrPX2xs9/fTTE//2t799b9myZZ++8MILG++7774lV1xxRSkdb6SfEm6knwpsdRxnoeu6q7szwA4TPllpCZ9kguH4lEv38OEN8l55rpvv+zHP86ap6nlAS9i1kt8XulbSusnTfx6pXSpNKV0qjSnnxHfZewyTmJeEXSuDRaQ0LFLo090ViURCioqKhhcWFiYTPiPWrl2b1djYOKq5uXl4NBpd5zhOWUtLS7Hv+2s4kPAZEN06xhhjjDnA4qZjU1RUNPhf//VfZy9ZsmT23r17R9xyyy0ls2fPXjhs2LA97d2jqsclEonJqnoJQef08nDDd2c3Lh1VHez7/kEb5CnFxW54mUfwOf2g4uJwClevGj2tqhHP8y7wfX8qQRFHdfh9re3tE9LSuskPOq4KyEy5dC/tFxd3yfFUGhzhNiGc5HUy0BQeR7Cys6aw9ZTNmzdn5ObmxoqLi2OVlZU5NTU1mZs2bcrZsWPHaNd190QikUrP80paW1tTi4vrDqe42BhjjOlptoFujDGmTSKyBLgUuEhVV6U8fibBGK2HVPXZI3zO14ELVHVcpy62l3vuuefOfvXVV58oLCz8zMSJE5vuu+++3KuuuqqEdjbS4YPO6ctU9QyCAPv9cAR5pwX1qjokJeGTrMbPIaiOTyZ8ErSf8OlzY7vD8Y9TwvGPEnZOL+npzuk2RuCnnrHX1vjA9KRPjxYthN0VF4TdFSOA+vD7WtGbuyu2bdsWyc3NzSkqKopVVlbmVFdXZ23cuDF7+/btY0SkNRqNVoXd5CUcSPjUqGqfn2Bgup+IfAK4BZgKnEBw7n0hwcSBQ557KyIjgWeAvwOOA1YB31LVd7ps0cYYYw7J4qbOsWLFikHPPvvsXYsXL35gx44dI7/85S+Xzp49e0FWVla7Halh5/Ql4WfQjPCz/aLO/Gyvqq6qjminuPi4lEt30nZx8fbe/Hm4LarqeJ73Ud/3pwFjgI3h97XHJ6SFI/Czw27y9J9Hajd5e8XFPVq04HneCb7vT1HVjwL7wykKBb1tikIq3/dZtWrVB8XFVVVVI+rr67O2bt06cv/+/VkZGRnr2ykujvf02k3fYzGTMaY3sQ10Y4wxbRKRnxGcL9VeIuhBVf3RET7ngEsEpfrxj398+muvvfZ4QUHBdWefffaOe++9N//aa69dRRDgt8n3/bGe580MR5BvdxxnUVipflibpWHCJ/Us7NQEw5CUS3fw4YRPo4js7OkkSVdQ1SGe500KxxQOFpGSsLticxe/biT155C2Ud7W+MAPNsrDooVe3TGf0l0xFTiJoPgj13XdVT3VXeH7PuXl5cMKCgpipaWlserq6hF1dXWZW7ZsGbl3795YNBptcF23vLW1tcTzvFLChA+wWe2DsulEIvJngvNt/wysJvjv/kHgY8CVqvpeB/dmAMsIxr5+HdgK3ANcC1yhqu936eKNMca0y+KmzhWPxzPmzp17x6JFix5qbGwce/PNN6+5//77F+Tk5Oxo7x5VzQg/208FhohIkeu67zuOc1gbeHrw0UjpxcUjgGT7dQsHPp/HgeSmbFN/OCIqnR44amw6cArB+10cFnZ3WfFu+PMY3k7MlN5N3l5xcZd0k3cW3/ezPM+7VFUvIpiiUOS6bq7jOFt7ak3btm2L5Ofn5xQVFeVUVFTk1NTUZG3YsCF7+/btYwAvGo1WqWppc3NzMQfOJq9WO3rKdCKLmYwxvYltoBtjjGmTiHwSeAv4hqo+lfL4HOCHwExVXXwEz3caQeXnm6p6fWevty958cUXx7/yyiuPFxQU3HDqqafu/trXvlZw3XXXLafjjfTRnufNUNVzCM7yXhye5Z1ISzCkn0uexYGETzMpCZ+U35v6+ui4oxV2Tl8YJtsyRaTScZxFjuPUH+2YwrTxgeln7LXVTZ6e9On1CZ/DEXZXTFXVjxB0VyyNRCIFItLuOM5jsXfvXqegoCB7xYoVsYqKilh1dXXm+vXrs5uamkb7vu9Go9EaoLS5uTm1m7xSVXt1UYLpP0RkpKpuTXtsKFAFFKvqlR3cezfwU4LzcAvCx1yC/6/uUtUpXbdyY4wxHbG4qWvE4/Howw8/fMuiRYu+3tDQcPKNN95YPmfOnHdHjRrVbnd5+Nn+4rBzeqiIrA430rckv67BWdgfjFtvo3tZCYpZ43w4dtrd20eZd5Xws/30sLB7V1gku+xYCgfa+HmkxrHp3eRtFRf3qhH4RyOconCx7/uXEhyXVhVupNd0xZ813/epqqoamp+fnywuzq6trc3csmXLyD179sSi0egm13UrEolESSKRSBYXlwGbrLjYdAeLmYwxvYltoBtjjGmXiLwGxzlX0AAAIABJREFUfBJ4EsgDJgGPAW+p6mfDa35JMF7pNFVdFz72FvAuUALsBs4DHgKGAtNUtbSb30qv9NJLL53whz/84bsFBQU3nXDCCfvuueeepTfccMMyx3HarOZX1YywA2AywdlpCYJN2MFANLws2b2cHFcXT46rA/YM1ITPoYRjCif6vj+doNp5XdiR3u4I8mQ3eTg+MH2jvK1u8g+Sb32hm7yzhN0Vk8PuCudYuyvq6+sH5+XlxUpKSmJVVVU5tbW1mZs2bYrt3r17lOu6Te2cs7deVft8UYLpn0RkPjAuLDZp75q3gBPDcZ+pjz9C8P/oE1W1oWtXaowxpj0WN3WdeDweefTRR7/4/vvvf339+vWn33DDDRUPPPDAu2PHjm1q6/pwIlK27/uXqOp5BLHSXoLP5cNSLk09Czt1ElePHo3U2/m+H/M8b1r4vW0Ji2Tz2yuSTSn2bqu4uK1u8vTi4u39obj4UFTV9TzvnPA4gjHA5rBIoeRo/jzu3bvXWbp06YiU4uKs9evXj2hqahqdSCQyMjIyqoE1YTd5anFxlxQ7G3OsLGYyxvQE20A3xhjTLhEZBHwHuBkYC2wEXiY4e6g1vOYl4MsEiaD68LFngSsJRjgPAbYA84HvqWpVeM104FFgCkFSYz3wG1V98hBr6nfnGb300ktj/vSnP32noKDgi7FYrPXzn//8mlGjRiXq6upGr1+/PufjH/+4Xn/99UMIKtKTkkmgoUBCRNY4jpPnOM7mgZBg6Cphwu1Mz/OmERQpbBWRZSKylXCMY5jwaWt8YProwH7TTd4Z0rsrWltbq15//fWa6667Ltd13YOubWlpkeXLl2ctX748VlZWFquurs5at25dVmNj4+jW1tahGRkZdSJS1tzcXKSqyYRPuaru7In3ZszREpHhwFpgnqre0MF1G4GFqnpj2uPXAP8DXKWq87p0scYYY9plcVPXi8fj7re//e0bFixY8M26urqzrrnmmtrLL798XUNDQ3Z9ff3I7du3H//zn/+8FcgGkh8uPWAPwfc2CmxyHKfAdd2y/tC93JN83x8eFsl+DBARWeU4TrmqDuLDxcWpxd7JbvL04mL7efDB2PxTw0leZwC7Fi1atObMM89cctJJJ33oGIP169cPzs3NjZWUlMSqq6uza2pqMjdt2hTbtWvXaNd1t7VTXLzOiotNX2IxkzGmp9gGujHGmG4nIjcDvwX+CPyBoNvidIJq0u91cF+/Os9IRM4l6E45C5jgOM5HVfV0VXUBBg8ezPjx4xM33njj1rvuuqsypZs8LiL7AXzfzww7AC4i2EjPj0QieZaAODLh+MDssJs82R0xjuDMQyd5GeE58aR0R4TnHtr3+zAluytee+21y++8886sCRMmeBMmTFiTSCSa6urqhjc0NOTs2LFjjIjsjUajlb7vr25paSnhwDl7taqa6OG3YUynEJGXgRuAS1V1ZQfXNQO/VNW70x6fDCwBblLVP3XpYo0xxnQ7i5s+2Dj5BDAh/HWW4zjn+r4/PHnN2LFj9eSTT97/pz/9afXgwYNTC1p3iIiG06bO8X1/JkEhbI3rugtd113bM++qb0rrJk/GTKMIikYGpVy6j6AQOb24eEB0k3cW3/dHtra2Tpk+ffqFmzdvZubMmY2RSKQ0Ho8Pqq+vH9HY2DiqpaVlWEZGxloRKWtpaSn2fT+1uLjd4w6M6UssZjLG9JRITy/AGGNM9xGR43r6vF8RGQe8CPxMVe9N+dKCw7j9K8BHOfg8o/cIuil+QNCV0ZfcB/wTUA+U+77/JlBx4oknbpg2bdr177zzzqebm5slIyOjXESWRCKRD51T7jjODsdxXvd9f6HneVNVdUpra+vkcJRebnuj9AYiPfis+NQz9trqJm8UkXoRWQ60+r5/GkHCbqiIlLiuW2Cb5ofH931KSkqOLywsTJ6zN2Lt2rVZW7Zs8SZOnOiLiLz22msTCRLCbxB0WBWqamPPrtyYriUiTwA3AV/rKBFkjDGme/WGmClch8VNgXHAX4FdBAWV5b7vv+m6buXnPve5U9euXfuF4uLic6ZNm7auoaFhxZlnnrkx/QlExI9EIsWqWuL7/kc8z5vped5tnufVu667oKvOm+6rUs4mTy0uTu8m9wi6yePAUmBbeO05wHARaXEcp8hxnLX2vT08jY2N0by8vJyioqJYZWVlrKamZvjGjRtzXNdtueCCC9x33303u7m5eQbBkQ//CbwG1O7fv/9DeQJj+guLmYwxPck60I0xZgARkS8Bc4EbkiMBe2AN3yE4D/CU5Nl/R3BvvzrPSERygL2q2uZG7Pz584c///zzj+Tm5t4ZjUajs2bNWjVr1qzFGRkZze09p6oel0gkpqjqJQTnTS8Lz/Le1VXvo7dJSfjkECRxkhvl6WeTJ8cHfjB6vaPxgWG3/9Sw219FZHl4lveHRukNRDt27Ijk5+dnr1q1KlZRURGrqakZvmHDhpxt27aNVlWNRqPVQGnaOXtVqtosIucADwBfAlqAT6vqop57N8Z0rZT/F35DVZ8+jOttHKExxnST3hAzheuwuAkQERcYBWzSNpKY8XhcnnrqqSvnz5//3TVr1lx89dVXr507d+6Cj3zkI+vbe85wTPYEz/NmEmzQb3Bdd6HjOBUDZbNXVaWD4uLhKZfuIWXcekpHeZvd5Cnd/tMJfm7rXdddFH5vB3wS2vd9SktLj1+6dOlBxcVbt24duXfv3hEZGRkbHccpD7vJ13AgbmokOIrgVmAOcCbwO1X9Us+9G2O6lsVMxpieZhvoxhgzgIjICcA6gmTQX3poDfOA84EvAk8DE4Em4L+Ah1W13Y3egfphePHixUN/9KMfPZybm3sXMOSf/umfiu++++6FgwYN6mgjfYjneZf4vj8ZiIrIijBx0S82e9PGB6afsddWN3l60ueoxweGRQqXqOqlQIaIFIdFCls74a31ar7vU1NTc1x+fn5s9erVserq6uy6urrMzZs3j9y9e/fIaDS6xXXdikQiUZxIJEoJxwcCG9tKeKYTkTHAXcAzqmrTE0y/FCaCvgM81tH43bR73iTY7Dgn7fHkRsgJqrqp0xdrjDEDUG+ImcJ1WNx0BOLxuDz77LOXv/322/9SUlJy6RVXXLFu7ty575933nl17d0TbqSf7nneZQTn0G8KN9LL+stmr6pmdFBcnN5N3lZx8f6jfF183z/T87zpwMlAo+M4i1zXLRERrzPeW2+2a9cuNz8/P3vlypWxysrKWHV1deaGDRuym5qaxqgqbRQXlwOVqnrI73dYVHIt4KjqX7v2nRjTMyxmMsb0BraBbowxA4SIOEAWsBL4s6o+KCKOqnbrGWQisgYYT9Bl+n0gj+Ac8MeBZao6s4N7B/R5RgsWLDju+eefn5Obm/u11tbW42+//faSu+++e8HQoUPbHSWuqoM8z5vk+/4UYLCIFLmu+77jOE3duPSj1k43efr4wNRu8g82yjvqJu+ktWV4nndR+L0dLiLlYVKo3W6XvmL//v1OYWFh1vLly2Pl5eWx6urqrPXr14+Ix+OjE4nEoIyMjBoRKdu/f38RBxI+Faq6u4eXbkyvJiLfBv4FeFxVv3sE990FvABMVtWl4WMuwSjeHao6rQuWa4wxA05viZnCtVjcdBTi8bg899xz095+++3Hi4qKpl922WUbHnjggfcvvvjimvbuCTd7Twk70k8FtjqO83642dvrE6cp3eRtFRe31U3eVnFxl71Pz/NO8n1/uqqeBex0HCfXdd3lItLSVa/ZXWpra4fk5eXFVq9eHauqqsqura1NFhePikajW1OKi1O7yTccTnGxMQOVxUzGmN7CNtCNMWYAEZEIsJzgw+PtqpoIH78UuA1YrKovd/EayoEzgEdU9ZmUx2cDzwGfVNV327l3wCaCUhUVFQ1+6qmn7l28ePF9e/fuHXHLLbeUzJ49e+GwYcPa7dhV1ajneR/zfX8qB87xfr83dE0fQTf5HlI6IlKSPkfdTd4ZVNX1PO9c3/enEYw8rA+7/St7+wjIDRs2DMrLy4uVlJTEqqqqcmpqaoZv2rQptnPnztGu6+6IRCKVnuetbm1tXc2BhE+9qvb7rhHTdUTkE8AtwFTgBIIzMwsJEiTLD3HvrcBLbXxJgbGquqWTl9tpRGQu8AzwfwSbHwdR1fzwul8SfH9OS47sFZEMYBlwPPAosAW4B/g74Ao78sAYYzpPb4iZwtezuOkYxONxef755ye98cYbT6xcufLyadOmNcyZM2fR5MmTqwg+N7Qp3OydqapnAE3hRnpRT8YbSand5GG8lBo3RcLLUrvJ04uLj6qbvLP4vj/K87xpqnousN9xnALXdQtEZG9PrutQ9u/f7yxbtuyg4uJ169ZlxePxMWFxcR2wpqWlpVhVkzFTRUdTIow5FIuZLGYyxvQ820A3xpgBQkRcVfVE5K/ASFWdHp7B/TAwG9gH/AfwZFcGeiKyBLgUuEhVV6U8fiZBF+tDqvpsO/cOuFGEHVmxYsWgZ5999q7Fixc/sGPHjpFf+tKX1tx3333vZWVltfvzU9WI53kXhmfSDReR0nAjvcvHWKV0k8eA9I3y1G7yuIjESemOcBynsacTPoeiqhKepTidIMDdEnakr+7JhFsikZBVq1YNLywsjJWVlcWqq6tH1NfXZzY2No5ubm4eHo1G6x3HKUs7Z69cVbf11JpN/yYifwZGAn8GVhMUnjwIfAy4UlXf6+DeW4FfEWxglKd9eVlvLu4QkXeBdrsFVdUNr3sJ+DJBMqg+5f6RwA8IRnYeR9Ad+a32Nk+MMcYcud4SM4Vrsbipkzz55JMXvf76608sX778U5deeumm+++/f8mMGTPK6WAj3ff9sZ7nzVTVs4Ht4ef6lV09fvwIusl3035xca9O9vq+n+l53hRVvRhQEVnuum5uTx831tDQMCgvLy+nuLg4VllZmVNbW5u5adOmnB07dox2XXd3WFxcklZcvLY3f/40fZfFTG2zmMkY051sA90YYwYYEfkJ8EngdeCzwBjg98C3VXVzN7z+z4A7aT8R9KCq/qide+08ozbE4/Ho3Llzv7Jo0aIHGxsbx910001r5syZsyAnJ6fdBETYNX1+uJE+QkQqHMdZ6LruhmNZS5jwOT6Z6EkmfQiCvbbGB6YnfXq0m7wzhCMgx4cb6WcQJNyWhAm31q563aampmhubm5OcXFxTmVlZU51dXXmxo0bc7Zv3z5GRFqj0Wil7/urW1paSjiQ8KlR1S5bkzFtEZGRqro17bGhQBVQrKpXdnBvMhk06VCdF8YYY8zR6umYKVyDxU2d7Omnn574t7/97XvLli379EUXXbT1/vvvz7388stL6XgjfbTneTPC7+Uux3EWh+PHj+kzdEo3efJM8va6ydsrLm4+ltfvDVT1OM/zLvF9/xJgkIgUu667uCunpCUSCSkqKhq+bNmynLKyslhVVdWItWvXZjU2No5qbm7OjEaj69KKi8sJiovjXbUmY9piMZMxxvQ820A3xph+TkQkeb6WiJwM3E3QQdECzAO+A6wMOy26/Hw/Efkk8BbwDVV9KuXxOcAPgZmquride+08ow7E4/Howw8/fMuiRYu+3tDQcPKNN95YPmfOnHdHjRq1vb17VNUJx4/PIEjWVLuuu9B13fr27gnvS+0mT90oT+0mTx0f+MFGeW8YH9hdfN8fE44pPAfY5zhOfjim8Kjev+/7lJeXDysoKIiVlpbGqqurR9TV1WVt2bJl5N69e3MyMjI2Oo5T3traWuJ5Xuo5e1vUPvSZXk5E5gPjVPUjHVxjySBjjDGdrrfFTOE6LG7qIs8999zZr7766hOFhYWfmThxYtN9992Xe9VVV5XQ8UZ6LNxIPxfYGxbIFkoH53indZOnFxcfn3LpbtovLu73n+FVNcPzvIt8359CMCWtIuz4X3e0z7lt27ZIfn5+zqpVq2IVFRU5NTU1mRs3bszZtm3bGBFJhMXFpWnFxdWq2ufPZTf9m8VMxhjTfWwD3RhjBgARcYCbgCcIgvVhwP2q+pPw69Le5loyQSQiEwjOXvrNsSaMROQ1go6OJ4E8YBLwGPCWqn42vMbOMzpK8Xg88o1vfOOmhQsXPrp+/frTrr/++soHHnjgvXHjxrVbNa+q4nneR33fnwmMAtY6jrMw7HbISU360H43eXrSZ0AkfA6H7/sjwjGFFwL+vn37lq9bt65o4sSJDW1dv2fPHjc/P3/EypUrYxUVFbHq6urM9evXZ2/btm2M7/tONBqtAUqbm5uLOZDwqVTVfd34tozpNCIyHFgLzFPVGzq4Lnme32aCkYY7gPeAx1R1dTcs1RhjTD/V22Km8HktbupCP/nJT8549dVXHy8oKPjc2WefvePee+/Nv/baa1cRHCvVpvBz/XRVvQBodhwnz3GcFcCwsJv8g+OqaLubvK3i4j7fTd4ZwilpE8MpaTGgvrS0dOW55567wnXdD13v+z6VlZVD8/PzY6WlpbGqqqrstWvXZm7ZsmXknj17YtFodJPruhWtra3FnueVciBu2mzFxaYvspjJGGO6l22gG2NMPycinyI4r+8agg/MLxFUon5BVV/tKBGU9jzPAnOA21T1t8e4pkEEXRw3A2OBjcDLwOPJcdJ2ntGxi8fj7mOPPXb9ggULvllbWzvhuuuuq5o7d+6Ck046aUvqdZ7nZYhIju/7yY7yU4FxHEj2wMHd5B9slA+kbvLOoKpDE4nEpb/5zW+mfPvb345cddVVTeedd97ibdu2aVVVVXZdXV3mpk2bYrt37x4ViUTirutWJhKJ4kQikZrw2dAdXU/GdCcReRm4AbhUVVd2cN1VwDSCTYSdwLkEGwMjgCmqWtINyzXGGNPP9MaYKXw+i5u6wYsvvjj+lVdeebygoOCG0047bdc999yz9LrrrltOyka653kiIpnJLnLf98cBp3BwYTEc6CZPLy7eYcXFh0dVxff9s3bv3j39kksuOTEnJydx1VVXFQ8dOrSypqYmu7q6OnPdunXZTU1NY3zfj0Sj0WpgTRvFxXt79p0Y07ksZjLGmO5lG+jGGNNPichUgkTKPwO1wDeBNwjGENYBv1LVRw7xHMlOirOAvwC7gMtUtfVwk0jdSUQuA9pKCClBkFDQwb3JCt227h2rqlva+FqfEI/Hnccee+xz8+bN+5e6uroJF1100bZx48a569evH7J27drBX/3qV+XOO+9MXr4bSJ6xh6qOJ6j+b3Bdd6HjOOWW+DkyLS0tsmzZsqwVK1bEysrKYtXV1VmNjY2xMWPGnLxmzZpIU1MTjuM0+L7/BvAOQcKnQlV39vDSjekWIvIE8A3ga6r6s6O4fzxQTNCJcV1nr88YY0z/NRBjJrC4qT0vvfTSCb/5zW+eLCws/EJmZmZi8uTJ+3fu3Dmovr7+uP3790fy8vKSlyYIi4vDz+wxgs10X0SWRiKRXBHZ00Nvo8+qr68fnJ+fHyspKYlVVVVlV1dXZ2ZmZo5PJBKZq1atgqDjv9D3/b8SfPYrB9ZZcbEZCCxmMsaY7hc59CXGGGP6GhER4CrgH4HvAz9W1cbwa1FgG8EYpw5HEaYEol8GTgYeCRNBrqp6Xfw2jsUjBJ0jqQ6nwlaB2wgC8VTtjj7vzUTkbuAyYAJwFjAEID8/P3bKKafI2LFjE1deeeXmM844o8x13SrHcRrTxweqKr7vn+p53kzP8/7R87wtjuMsdF231DbSD7Zly5aM3NzcnOLi4lhlZWVOTU1NZkNDQ87OnTvHOI6zJxKJVPq+v7qlpeUtoLy6urosKytrI3Cz7/sPA7cTJN6+YJvnZqAQke8QJIK+cTSJIABVXSsii4DJnbo4Y4wx/ZrFTIDFTYjIR4F7CWKmswk6/dmzZw/z5s07/pRTTtEJEybsOf3009cDyyKRSJvd5Kp6XCKRmKKql7S2tl4qIstc113sOM6ubn9TvVhLS4usXLkyc9myZR8UF9fX149obGwc3draOjQjI6NORMqam5sXqWqym7zcdd1TPM972Pf9Gwl+Vver6ls9+26M6R4WMxljTM+wDXRjjOmHwuTOd0Tk2dSNOBGJhMmcCoKgk/YSQSmdFKcBnwWqgV+HX+7tFd5VHXVNHMJqVV3eqavpORcSnGeeR/CzKwfKpkyZUj916tRPzp8//7uvvPLKxTt37jzxpJNOqjnnnHM+dPaeiOC6bq3rurWe553sed5M3/ev930/7jjO+67rFotIb//z0Gl836e4uHj40qVLY2vWrIlVV1ePWLt2bVZjY+PIffv2ZWVkZKx3HKe8tbW10PO8NYQJH8/zGjt42l+Eozc/C3weaOqO92JMTwsTQd8hOIvv6Z5ejzHGmIHFYibA4iYIzomfShAr/SL8vQyo+MUvfjH417/+9TeXLFlyW2lp6ehx48aNve222+raKiQWkb3RaHS+qi7xPO9S3/cvTSQSHxORFa7rLnIcZ0f3vq2etWXLloy8vLwPiotra2uHb9y4MWfHjh1jRGRfNBqtCouL53Ng7Hrt/v37E+085SrgiyLyLWAu0FF8ZUy/YTGTMcb0HBvhbowx/Vxb3RLhZt1U4GOq2mZFfPK+8MP6AwTn5f20o06Knh5RmDKK8HpV/a8jvPdWgnMOJ/WTRNAhxeNx+eEPf/jxefPmPV5SUnLpFVdcsW7u3Lnvn3feeXUd3ed53gm+789Q1QnANsdxFrmuu0pEenuHzWHbsWNHJC8vL3vVqlWxysrKnOrq6syNGzfmbNu2bbSq+hkZGdWqWpp2zl61qn6oCMEY82Ei8m3gXwjOcP3uMT7XaQRJ1TdV9fpOWJ4xxpgBZiDFTOEaLG46Av/3f/834mc/+9k38vLy7hg6dKjMmjVr5R133LEkEom0tnePqg7yPG+S7/tTgMEiUuS67vuO4/SbYlnf9ykpKTl+2bJlsdLS0g+Ki7ds2TJy3759IzIyMjY6jlPW2tpanFJcXJac9GCM6ZjFTMYY07NsA90YYwYgEfln4GfAqLaC15ROivHAf4cPT1PVPYdK+ISjEJ2eGFeYkgjaTHAO3V4gF3hCVRcf4t7kWX6bCUY17iAYZ/iYqq7uwmX3uHg8Lv/2b/827c033/yXoqKiGTNnztwwd+7c9y+++OKaju7zfX+M53kzVPWjwM5wI32FiLTXNdCr+L5PTU3Ncfn5+bHVq1fHqqqqsuvq6jI3b948cs+ePSOj0ehm13XLE4lESSKRKCXsJgcaejrpafo+ETkfeBI4l+DvnH0Ef75eUNXfHcb9I4FngL8DjiNIhnxLVd/pskV3EhGZS7D2/wMeT/+6quaH1/0SuAU4TVXXhY+9RfD3fAmwGzgPeAgYSvD/qdLueA/GGGP6v/4aM4Wvb3HTUZg/f/7w559//pHc3Nw7o9FodNasWatmzZq1OCMjo90iWlWNep73Md/3pwJDRaQk3Ejf2o1LPya7du1y8/Pzs1euXBmrqKiIVVdXZ27YsCF727ZtY1RVo9FoNZAsLk528Fep6v6eXbnpDwZq3GQxkzHG9DzbQDfGmAFIRIYCDwLz2kqQpHRSfIvgQ/Z3VfW59jopRGQccBFBB+6a5HMQ/H+m20YXisgFBIHDewTn750Rrn8CcI2qvt3BvVcB0wjGne8kCM4eBUYAU1T1cM4C7NPi8bg8//zzk954440nVq5cefnUqVMbHnjggUWTJ0+uIjjnsE2+748MN9InAnscx1nium6hiLTbkdGd9u/f7xQUFIxYsWLFBwmfdevWjWhqahqTSCQyMjIyqoE1zc3NJRzoJq9Q1T09u3LTn4WJ638EFgEbCJIZXwRuIkjofL+DezOAZcBw4OvAVuAe4FrgClV9v2tXf2xE5F1gZntfV1U3vO4lgvNkT1PV+vCxZ4ErgZOAIcAWYD7wPVWt6uKlG2OMGUD6a8wUvq7FTccgLy9v2DPPPPNQbm7uXcCQO+64o/irX/3qwkGDBnW0kR7xPO9C3/enA8NFpDTcSN/UfSvvWHV1dVvFxbHdu3ePikajW1zXrUwkEsUpxcVlwEYrLjZdaaDGTRYzGWNMz7MNdGOMMQdJ6aQ4EXgNiAIzVHV7O6MNMwjObf5H4HxgLfCoqua28/zHqerern0XB71eJlAMNKrqRUd47/jw3nmqel1XrK+3evLJJy96/fXXn1ixYsUnJ02atGXOnDmLZ8yYUU7HG+nZnudNV9Xzgf2O4+S6rrtURLplrPn69esH5+Xl5ZSUlMSqqqpyamtrhzc0NMR27do12nXd7ZFIpMrzvJLW1tbVHEj41Hd3wtKYjohILjBWVU/p4Jq7gZ8SJKkLwsdcgm6KXao6pTvWaowxxgxU/S1mCl/T4qYjtGDBguNeeOGFOUuWLPlaIpEYdvvtt5fcc889C4cMGbKvvXtU1fU873zf92cAWSJS4TjOQtd1N3THmvfv3+8UFhZmLV++PFlcnLVu3bqseDw+JpFIDMrIyKgVkbL9+/cXcXBx8e7uWJ8xh8viJmOMMV3NNtCNMcYcJCUZ9BBBJ8H3VfWHbXVSpFwbBU4ETgb+meCswJ+q6rNp159McF7eKuCh7tq4FJF/D9d13JGeUS0irwMXqOq4LllcL/f0009P/Nvf/vbEsmXLrrnwwgsb77vvviVXXHFFKR1vpGd5njdNVS8EWh3HyXddN09EjnmEXyKRkBUrVmQuX748p6ys7INz9hobG0e1tLQMj0ajax3HKWtpaSn2fT95zl65qm4/1tc2pjuIyP8AH1XV0zu45i3gxPD4hNTHHyEYb3iiqjZ07UqNMcaYgas/xkzha1vcdBSKiooGP/XUU/cuXrz4vr1794645ZZbSmbPnr1w2LBh7U60UlXH87xzw430HKDadd2FruvWd8aaGhoaBuXl5eUUFxfHKisrc2prazMbGhpydu7cOcZ13Z2RSKSyneLiHjlWwJgjZXGTMcaYrmYb6MYYYz5EREaCoZCFAAAgAElEQVQD+QSB/DXpY62SiSERGQKcA3yE4FzoeeHXv0vQYXEVKedFi8h1wJ8Jxmw91Y3vJ5kIGqKqLUd474BNBKV67rnnzv7v//7vx5cuXfr3EydO3DZ79uzcq6++upiON9KP9zxvqqp+DPBFpCASieSKyCG7aZqamqK5ubk5RUVFscrKypyamprMjRs35mzfvn20iLREo9FK3/dXt7S0JMeulwM1qtorxsYbc7iSZ6ASjD39AvBjYLaq/ryDezYCC1X1xrTHrwH+B7gq+fexMcYYY7pGf4uZwte2uOkYrFixYtCzzz5715IlS+Zs37591Je//OXS2bNnL8jKytrV3j2qKp7nfdT3/ZnAKGCt67oLHMepDT4mti+RSEhRUdHwwsLCWFlZWayqqiqrvr5+RGNj46jm5ubh0Wi0Pq24uJyguLipU9+4Md3A4iZjjDHdzTbQjTHGtElEvkZQkbsLeBn4A8Hotn0p1/wMuB3YBGSED38rvH4v8ClVfTe8dizwIvBRYLqqbmrvfMBOfh8jgCJgc7iReyT3nkbQ+fGmql7fFevra3784x+f/tprrz2+dOnSz5111lk7Z8+enX/ttdeuAtrtjFHVoYlEYoqqTnrwwQcjqrrp7rvvfm3ChAmby8rKhhUUFMRKS0tj1dXVI+rq6rK2bt06cu/evdkZGRkbHcepaG1tLfE8L/Wcva12zp7pL8K/R/85/NcEQafZvx3inmbgl6p6d9rjk4ElwE2q+qeuWK8xxhhjDugvMVP42hY3dZJ4PJ4xd+7cOxYtWvRgY2PjuJtvvnnN/fffvyAnJ2dHe/eoqvi+P8HzvJlFRUVjH3nkkZZbb7118c0337xw586dkfTi4g0bNmRv3759rIi0hsXFpSnFxWUExcVHVARhTG9mcZMxxpjuZhvoxhhjOiQijwIPEwQovwH+qKqFInIuQZLkX4G/As3A54C7CCqC9xAEI2+Hz3Md8HtgNvBfqtqY8hpOZ4wmFJGXgTpgGdAEnAU8AJwGXJ2SmPolcAtwmqquCx97C3gXKAF2A+cBDwFDgWmqWnqs6+tPXnzxxfGvvPLK4wUFBTeceuqpu+65556ln//855fTxkb6nj173Ly8vOyKiopxhYWFl73zzjsj9u3bh+d5CSCRkZFRBaxpbm4u5kDCpyo18WhMfxWenToq/PUZgr9DH1XVH3RwjyWCjDHGmF6kL8VM4XNZ3NQN4vF49OGHH75l0aJFX29oaDj5xhtvLJ8zZ867o0aN+tDxUr7vU15ePqygoCC2du3aj82fP//s1atXu4MGDUo0Nzc7kUikIRKJlLdRXLzFiovNQGBxkzHGmO5mG+jGGGMOi4g8CHwDWAvMAO4lOO/v46q6XEREVVVEsgk6F8qAu1W1Igx0ngBuBV4FjgNiwA9V9Y+duMaHgX8ETgWGESSD3gf+VVWXp1z3EvBlgkRQffjYs8CVwEnAEGALMB/4nqpWichlBImidApMUdWCQ6xtJPAM8HcE738VwVjGd47+Hfe8l1566YQ//OEP3ykoKLh53Lhx+2bMmFHuOE68uro6u7a2NnPz5s2x3bt3j4pEInHXdSsSiUTx6NGjG/bv339OPB6/kuDn9BuCn1FND78dY3pcODr1DuCE1KR52jU2itAYY4zphfpCzBSu0+KmbhSPxyPf/OY3b16wYMEj69evP+3KK6+sGz9+/Jr169cfX11dnbl+/frspqam0b7vu9FotAYobW5uLj7//PNHr1q1ahIwiaBg4XvA/7Nzyo2xuMkYY0zXsw10Y4wxR0REPq6q74nIp4C/APcBv00G8SJyP/AjgiTH98PH/h74LRAHfglUA5cAdwLfVdVnu/+dHJmURNAjwHtpXy5R1XbP9RaRDILujuHA14GtwD3AtcAV6ecl9kW///3vR//qV7/6l3fffff2aDT6VktLy0pVTZ5NXq6qHzr3T0SGEVSNP0hQFf7Nbl62Mb2OiNxG8PdkuwlmEXkTOFFVz0l7/BGCMbInqOqmrl6rMcYYY9o2UGMmsLipI/F43H3ssceu/93vfvejPXv27BORea2trand5OvbmjIgItMJxv5PAk5pK7YyZqCxuMkYY0xXsw10Y4wxR0VEYsAvgCkEHcRbgOnAJwmC/1tUtVRExgHPATOBm1PGAQ4Hfgd8hCDg2drGa3TamMJjlZIIul5V/+sI770b+CkpgZ2IuATdFLtUdUpnr7cvEZEhQMQSQcaAiPwWuAkYo6rxdq65C3gBmKyqS8PHkn+n7FDVad21XmOMMca0b6DFTGBxU1cSkVGquqWn12FMb2BxkzHGmK7m9PQCjDHG9E2q2qiqnyXoLJgKDALOBhqBv6acfXcRwfi9/yQYC4iIuKq6E3iZ4Jy9oe28hh9eH+nCt9IdPkvQhf1BVXTYffIycImIjO2xlfUCqrrPNs8HHhE5X0T+JiJrRWSviMRFZImIfPEw7r1VRPw2fnkiMqo71n+sRORFEXlGRG4QkZkicp2I/BH4IsGo1nh43S9FpFVETkq5/VdAKfCKiNwkIlcArwBnEnRrGWOMMaYXsJjpiFnc1AHbPB+YLG6yuMkYY0zP6A8fro0xxvQgVX0JeAlARE4jOOvvrfDfxwH/AGQAi4D0s9ouBXYAJwN14UjvaeHjJwLLVfXnqprohrdyuF4QkT8Be4Fc4AlVXXyIeyYCC9t4vCj8/RygofOWaEyfkAXUA78HNhAkhb8I/H8iMj45zrQDCtxGcExAqja7D3qhJcDtwC0E34vdBJ0QX1LVP6Rc5wAS/gJAVVtE5BPAD4CfEJwPuhK4WlUXdc/yjTHGGHO4BmDMBBY3GdNZLG6yuMkYY0wPsBHuxhhjOpWInKWqFeE/fxb4N4LEzqdU9Z2wk8ITkZGESSPgclXdLiJ/Aa4BtgFLgfMJir3uVtXX0l5nPPBt4Peq+k43vK8LCAK29wgCzTOAh4AJwDWq+nYH9zYTnPF9d9rjkwmCwZtU9U9dtHRj+hQRyQXGquopHVxzK0E3wSRVXd5dazPGGGOM6Qz9NWYKX9PiJmO6gcVNxhhjTNeyEe7GGGM6VUoiKApcAJwE/DfwDyIyPBzBB8EYwwnAH8NE0DcIOi++pqrjCJIulwOLgTkikp32UpOBfyJINHU5VV2pqg+o6muqulhVf0PQ+dEAPN0dazBmgGjkw51XxhhjjDH9Rn+NmcDiJmO6kcVNxhhjTBeyEe7GGGO6hKq2At8Vkf8AxgO/AIaJSDlBAuUzwJ+A/xSRk4EbCAq7Tgvv3wHsEJEfEySEpgD/CyAiJxCM8FpJMMasR6jqDhH5G/DPIjJIVZvbuTQOpCezCB9T+s7oNGM6nYgIwX/7I4AvAFcBsw/nVuB/w86sHQRdTo+p6uouWqoxxhhjTKcaCDETWNxkTGewuMkYY4zpXtaBbowxpkup6kZVzQXuJjjT7n7gVOBJgs6JJoKuivOBF4CbRWS3iCTP8aoFmoCzU552KvAp4NledNZfR2eiFAPntvH4eeHvJZ2/HGP6jH8HWoEtwI+Bh1T154e4ZxPwPeAO4OPAt4BJQJ6ITOy6pRpjjDHGdL4BFDOBxU3GHC2Lm4wxxphuZGegG2OM6VYichKwKey2SD72ZeAZYAawliC4ewA4AcgjCPQ+o6r/G97/c+AU4EJVbenWN5BCREYARcBmVf1YB9fdRZDomqyqS8PHXGAVsENVp3XHeo3pjUTkRGBU+OszwF3Ao6r6gyN8nvEESdd5qnpdpy/UGGOMMaab9KeYCSxuMqYzWNxkjDHGdC/bQDfGGNPjROQjwArgEVX9cfjYcOA6gqTQEOAsVVURuY5gjOGdqvpSN67xZaAOWEbQ3XFWuLbTgKtV9d3wul8SnEV4mqquCx/LCO87HniUoGL8HuDvgCtUdVF3vQ9jejsR+XeChPAJqtp4hPe+DlwQnglqjDHGGNNv9IWYKVyTxU3GdAOLm4wxxpiuZSPcjTHG9DhVXUMwVuxmEfmMiBwH7FLVXxOc43d1mAgaC9wK1AB/6OZlFgGfBn4JvA08QTBCcEoyCRRyCM4Yk+QDYcfHJ4B3gZ8ArwGjCd6XJYGMOVgBECE829MYY4wxxvSZmAksbjKmu1jcZIwxxnQh60A3xhjTK4jIKIKRhP8IzANeB+LAm6q6Pbzm74H/R3AO4H/01Fq7i4h8BfgPYLeqDj/EtbcCbXWXKDBWVbd0wRKN6XQi8lvgJmCMqsaP4L7TCMZ7vqmq13fV+no7Ebkc+D7whKq+3tPrMcYYY0znsZjpwyxmMgOVxU3HxuImY4wxh2Id6MYYY3oFVd2iqrcCFwGNwI3AtcDp8EGy6BZgHfByT62zu4jIOILk2IYjuE0Juk0mp/yaQpBUM72ciHxFRHwR2XmY148UkV+LyFYR2SMiS0TkE129zs4iIi+KyDMicoOIzBSR60Tkj8AXgR8mk0Ai8ksRaQ3P8kze+5aIPBp2X10uIvcBC4EE8FiPvKEeJiLJz/XHAROAc3twOcYYY4zpAhYzHcxipoFnoMVMYHFTZ7O4yRhjzOGK9PQCjDHGmFSqWgrcJiLZwNDkeXjAJcBVwNdVdW+PLbD7vAi8B2wHPn8E961W1eVdsiLTZdKSf5mHcX0G8A4wHLgX2EpwPuQbInKFqr7fhcvtLEuA2wmSvFnAboJOiC+pauq40Q+N9wSKgZuBkwjO+9xC0IX1PVWt6vql9xwREQ58Tzw9ME4q+XsdwZ+js7p/dcYYY4zpDhYzfcBipgFkgMZMYHHTUbG4yRhjzLGyEe7GGGN6PRE5nuAMvb8HYqq6u4eX1KVE5EvAT4FzgCeBzx/mOMJfAZMsGdT3iMj/EHQBbOfwft53E/wZmaKqBeFjLkEiZZeqTuniJZtOICIx4G5gq6r+TEQiqpoQEdEj/JCevEdERgJ/BhxVvawr1m2MMcaY3sdiJouZ+juLmQYui5uMMcb0BBvhbowxptdT1V0EFeOfHgCJoNHAc8AjqrrxSG8H/ldEEiISF5G/iMg5nb9K05nC5N90gm6Iw/VZoDyZCAJQVY9gVOclIjK2c1dpOlPYDQEwgiARdFvq15NJoJTxgqn3niQij4jIuyJSLCL/KSLnpSSOmoCNwLguewPGGGOM6XUsZjr827GYqc+xmGlgsrjJGGNMT7INdGOMMX2Cqm5W1Xd7eh3d4AWgVFVfPML7NgHfA+4APg58C5gE5InIxE5doek0x5D8mwgUtfF48jFLAvYSEnDDXwIHEj0E/90uA3LCxxMicoKITBeRc1XVT3uuUQR/Xr4AlAJ/AC4FlovIZWE3hUeQCMoUkTHd8iaNMcYY0ytYzHRIFjP1QRYzDQwWNxljjOlt7Ax0Y4wxppcQkeuBvwPOP9J7VfVN4M2UhxaJyOsEZ549DlzXKYs0ne1ok385BBXz6ZoIumpyjnVh5sB4v7THcoB/Bd5U1b8camxg+DWvna/tEpGNwOXhCMEXgU8RjKZsEpGfAz8KkzsQJHw/CcwC/jd83h8CC8M1fRGoJTjLLwKcQZBsMsYYY4zpFyxmGpAsZurlLG4yxhjTH1kHujHGGNMLiMhQgvPZfgpsFpFMEckCMsKvZ4rIcUfynKq6FlgETO7s9Zpjl5L8m9XTazEHE5FPi8ht7SR4WoCvEHQt0VESSESOF5HJInKfiDwlIrNEZFDaZRsAH3g//OdPANcQdEo8DdwcPtcIgj8vP1bVV1R1r6o2q2oL8B5wCUEHFQSdFK3A2Uf+7o0xxhhjeieLmQYei5l6N4ubjDHG9Ge2gW6MMcb0DjFgNDAX2Bb+agJuAoaF//5yj63OdKpOSP7Fgew2Hs8GNPy6aUeYoDkn/GdJ+9rpBF0KF7Vxn4Tni8aBmIgM7uA1pgK/AX5HMCb0HOAhgk6nc1Mu3UDQOREHnlHVpaqaC3wZKADuDrssTgYGAeeLyOPhWX7rRGQ/MAdYy4HP9uuAXcBHjuT7YowxxhjTy1nMNIBYzNTzLG4yxhgzkNkId2OMMaZ32ERYmZ3mUWAmcDVHGOCLyGnADA4eU2h6h9Tk34NtfH0b8Crtj5EsBs5t4/Hzwt9LjnWB/dzPgZtE5Cuq+isAEXHCs/O+DTQQjPFM5xCM/6sHTgVGhNd+IGU04ViCRM5jqloSfm0c8FfgfhH5Z1VNECRwXKBEVetFxAVQ1e0i8t/AYwQjBbcSdEhcAwwGKoH/B6wJn2MrQfIHguRSI3DWMX2XjDHGGGN6F4uZBhaLmXqexU3GGGMGLNtAN8YYY3oBVW0mOI/rICJyO+Cp6vspj/0SuAU4TVXXhY+9BbxLkATYTZAUeIigQvuxLn8D5kgda/Lvr8ALIjJJVZcChAmELwF5qmrnt3VAVb8oIh7w7yKyD/ijqvoichbBuL8XVLWxjVuTXRergcuBkUBD6nl+KaMJX1VVT0TGi8htBKMC///27i1Us6sw4Ph/xUvUkAtIvYdYRaxYG0VEpVbEC6S0WDGCIEUNSEFTb4hgEBQ0PmjxQsSHPKggFqoRqRa1akpFYutTqcakNYyNicZ4jbdUimNm+bD3iWeOk0ymyXznJN/vB4c52dnfOYszZx7W+n9rr8e1PC7wcMvizn+3LCTd1LI4uPM9juz6Pqe2LB5+rfpWddac88+PGtSyG+Qh633/0/K7873q0f+fnw8AwEFkzrR1zJn2mXkTANtMQAeAg2/vWWGntEwWdz9C7cqWM7/Oru5f/bC6vLp4znloE4Pkjruzi3/Vh6oLq8vGGBe1/H1fWD2mes5JHv7d2q5FmwurB1SXtCy8fKzlbMVTqsv23Ltj5/P/rF7cslvi6y3/Fo/6d7ouAr2getd66VvVv61/vqh64Hr9By07H/54fd1vdn2ZM3bumXP+aozx3uqfxhgfbfkduL46rWVx6bzqi9Wlc87/HWPcXD14jPGAOeevTvgHBQBw92LOdA9jzrS/zJsA2HbOQAeAA2zOecGc88xjXLv3nPP6XdfeMOd8wpzzrDnnqXPOs+ecLz/oC0FjjFeMMY6MMX5xB+592Xrv3o9bxhgP2sR4N+S4i39zzl9Xz27ZQXNJ9emWd+KfN+e8YkPjvFuac871sYO/rF7d8rjA94wx3lw9r/rEnPOqMca99ywC1e/+br5e3ad62J7rt1ofB3pZdXX1l9VfzTnfXH2yOr3a+Z29qeVRgo8cY7xk52zBMcbDq1e2LBJdvY79y9WrWnZIvLu6ovr36k3rff+yawivrR5hEQgAuKczZzrqXnOmnRvMme4U8yYAtp0d6ADAvljPNfu7lgnsmce5fcesXl59c8/1Ezrr8KCac15QXXC8a+v1Hx3rOse3PnZwzDlvHGO8smVR5e3VtS2La3t3NNz6uvXT/1r/PHu9/nsLQdUftSzkXTTnvGbX9We1LOw9dIxxrznnb8YY17c8RvSS6kljjO9Uz23ZHfOaOefNdevOjr8fY3yuenL16+qaOedR5wmuY/rpHf+JAABwEJkz/T5zps0xbwJgmwnoAMB+ubT6UvWz6vwTeN1Vc87/OCkjYmvsOnvv+jHGG6qnVjetuyj2PoJw72tvGGMcrs5ZF3NuOcZtP285T/O5Y4wbW87v+4tq5xy+R7Y8CvGX68cZLTs7HlT9bfXt6nXVx3ePeR3bTS2PHQQA4J7NnIl9Zd4EwLYS0AGAjRtj/HX1jOrx1Tv2eThsuXUx6MrqMcc7+259jOGR6rstOylOr342xrhvy+MJT6/GnPMrY4x/rt7Z8ojDw9U5Lb/vr18/v9f6Zf+h5ZGCX5lz/l/11tsZ620uUAEAcM9hzsRBY94EwDYR0AGAjRpjPLh6b/WmOef31qPL7vDLq8+MMf6g5Z3qX6reMue86i4fKFth166J71ZPqR5RXXM7uylOqY5UV1bnVs8fY9xSPaHljL1zq0NjjNdUL67+puXsxR9XH6k+V31yPZNx5/sfqg702ZsAAGyOORMHjXkTANtGQAcANu0D1dVzzktP8HXfry6uvlr9omXifVH11THG0+ec37hrh8mWGC3nRF5bndqymHPNruu35drq+dWHW3ZJ3NByxt+nqsurG9ZdEZesH8dkZwQAAMdgzsRBY94EwFYR0AGAjRljvKjlPLNzT/S1c87PV5/fdemKMcZnW97R/rbqhXfJINlWh1rO3ntsy26H29rmc2T9830tiz5XzTl/dPKHBwDANjBn4oAzbwJgKwjoAMBGjDFOq96/fvxgjHFmy2T7vuv/P7M6fHvnqO0157xujHFF9bSTMGS2y43V/ao/XP/7yLFuWs/xa855XXXdZoYGAMA2MGfibsC8CYCtMDz9BADYhDHGOS2Pb5vd9rvU/3HOeUK7ItYdFU+ccz7sTg6RLTbGuE91ll0RAADsF3MmDjrzJgC2hR3oAMCmfL961jGuX1Q9szqv+smJfMExxqOqP+voxxTCCZtzHq4sAgEAsJ/MmTjQzJsA2BZ2oAMA+2qM8eHq/DnnGbuufbB6afWoOed31mtfqP61+kZ1c/Un1Rur06o/nXNevemxAwAAnGzmTAAAm2UHOgBwEOx9R98pLY8s3P3Ywiurl1RnV/evflhdXl085zy0iUECAADsE3MmAIANsQMdAAAAAAAAAFreqQgAAAAAAAAAW09ABwAAAAAAAIAEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAFRsUR8AAABGSURBVAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKgEdAAAAAAAAACoBHQAAAAAAAAAqAR0AAAAAAAAAKjqt9oOO95LcpwRAAAAAElFTkSuQmCC","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536408893E12,"submitTime":1.465536403984E12,"finishTime":1.465536409945E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"c430d55c-9410-4e3a-8b26-06a6087b443c"},{"version":"CommandV1","origId":124233,"guid":"ab7a4715-f810-4001-815d-2c7d7c7267a5","subtype":"command","commandType":"auto","position":2.671875,"command":"%md\n###Section 2. High Order Spectral Clustering","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.46553640399E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"63648c56-6a70-404a-9dfd-0f3b013227e0"},{"version":"CommandV1","origId":124234,"guid":"af3183f1-9479-4c52-9090-472183213af3","subtype":"command","commandType":"auto","position":2.7109375,"command":"import tensorSVD.{TensorSVD, RddMatrix, RddTensor}\nimport highOrderSpectralClustering.graph.ConstructGraph\nimport highOrderSpectralClustering.clustering.TensorSpectralClustering","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">import tensorSVD.{TensorSVD, RddMatrix, RddTensor}\nimport highOrderSpectralClustering.graph.ConstructGraph\nimport highOrderSpectralClustering.clustering.TensorSpectralClustering\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"<div class=\"ansiout\"><console>:40: error: not found: value tensorSpectralClustering\n import tensorSpectralClustering.clustering.TensorSpectralClustering\n ^\n</div>","error":null,"startTime":1.465536409949E12,"submitTime":1.465536403996E12,"finishTime":1.465536410024E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"2b52ce97-ffbe-438c-954b-ccfc2049489f"},{"version":"CommandV1","origId":124235,"guid":"51ccd322-c187-4adc-ba2a-3e21395fd63a","subtype":"command","commandType":"auto","position":2.75,"command":"def getSpectralCluster(feadf:sql.DataFrame,feaNames:Array[String], idcolName: String,\n nc:Int,K:Array[Int],sc:SparkContext,sqlContext: SQLContext ):sql.DataFrame={\n\n val featureSelected: sql.DataFrame = feadf.select(feaNames.head, feaNames.tail: _*).cache()\n val mycluster= TensorSpectralClustering(featureSelected,nc,K,sc)\n val isLocalTensor=true //false means use distributed tensor method instead which is suggested for larger sample.\n mycluster.run(isLocalTensor,sc)\n\n import sqlContext.implicits._\n val labeldf = sc.makeRDD(mycluster.labels).toDF(\"label\", \"id\").cache()\n\n labeldf.registerTempTable(\"clusterLabel\")\n feadf.registerTempTable(\"fisheriris\")\n\n val fisheririsCluster=sqlContext.sql(\n \"SELECT fisheriris.sepalLength, fisheriris.sepalWidth, \" +\n \"fisheriris.petalLength, fisheriris.petalWidth, \" +\n \"fisheriris.id, fisheriris.specie, clusterLabel.label FROM fisheriris, clusterLabel \" +\n \"WHERE fisheriris.id=clusterLabel.id\"\n ).orderBy(\"id\")\n\n //get unique class\n val groupCounts=fisheririsCluster.select(\"specie\",\"label\").groupBy(\"specie\",\"label\").count().show()\n\n fisheririsCluster\n }","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">getSpectralCluster: (feadf: org.apache.spark.sql.DataFrame, feaNames: Array[String], idcolName: String, nc: Int, K: Array[Int], sc: org.apache.spark.SparkContext, sqlContext: org.apache.spark.sql.SQLContext)org.apache.spark.sql.DataFrame\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536410031E12,"submitTime":1.465536404001E12,"finishTime":1.46553641024E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"504f1add-f0c0-4d48-a078-fdc3c3b88141"},{"version":"CommandV1","origId":124236,"guid":"e339d97d-472c-41e3-a031-442631d77f71","subtype":"command","commandType":"auto","position":3.0,"command":" val K=Array(5,5,4)\n val nc=3\n val HOSpectralCluster=getSpectralCluster(fisherDFwithIndex,feasName,\"id\",nc,K,sc,sqlContext)","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\">+------------+-----+-----+\n| specie|label|count|\n+------------+-----+-----+\n|'versicolor'| 1| 48|\n|'versicolor'| 2| 2|\n| 'setosa'| 0| 50|\n| 'virginica'| 1| 4|\n| 'virginica'| 2| 46|\n+------------+-----+-----+\n\nK: Array[Int] = Array(5, 5, 4)\nnc: Int = 3\nHOSpectralCluster: org.apache.spark.sql.DataFrame = [sepalLength: double, sepalWidth: double, petalLength: double, petalWidth: double, id: bigint, specie: string, label: int]\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536410245E12,"submitTime":1.465536404007E12,"finishTime":1.465536416388E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"110a698c-821b-408c-86f1-094d4ae22cf4"},{"version":"CommandV1","origId":124237,"guid":"6ddc8bf2-a2ad-42a0-ba52-2db5f80590d3","subtype":"command","commandType":"auto","position":3.5,"command":"%md\nHigh order spectral clustering error ratio 4%. It shows clearly that the high order spectral clustering done a better job in separating \"versicolor\" and \"virginica\" which seems hard to be distinguished by k-means in the original feature space.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536404013E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"38295c25-f7e0-4b62-a038-b05cefce23ad"},{"version":"CommandV1","origId":124238,"guid":"6313bcb8-ad0e-46e5-a3b5-9a45b52306a2","subtype":"command","commandType":"auto","position":3.75,"command":"HOSpectralCluster.registerTempTable(\"HOSpectralTable\")\ndisplay(HOSpectralCluster)","commandVersion":0,"state":"finished","results":{"type":"table","data":[[5.1,3.5,1.4,0.2,0.0,"'setosa'",0.0],[4.9,3.0,1.4,0.2,1.0,"'setosa'",0.0],[4.7,3.2,1.3,0.2,2.0,"'setosa'",0.0],[4.6,3.1,1.5,0.2,3.0,"'setosa'",0.0],[5.0,3.6,1.4,0.2,4.0,"'setosa'",0.0],[5.4,3.9,1.7,0.4,5.0,"'setosa'",0.0],[4.6,3.4,1.4,0.3,6.0,"'setosa'",0.0],[5.0,3.4,1.5,0.2,7.0,"'setosa'",0.0],[4.4,2.9,1.4,0.2,8.0,"'setosa'",0.0],[4.9,3.1,1.5,0.1,9.0,"'setosa'",0.0],[5.4,3.7,1.5,0.2,10.0,"'setosa'",0.0],[4.8,3.4,1.6,0.2,11.0,"'setosa'",0.0],[4.8,3.0,1.4,0.1,12.0,"'setosa'",0.0],[4.3,3.0,1.1,0.1,13.0,"'setosa'",0.0],[5.8,4.0,1.2,0.2,14.0,"'setosa'",0.0],[5.7,4.4,1.5,0.4,15.0,"'setosa'",0.0],[5.4,3.9,1.3,0.4,16.0,"'setosa'",0.0],[5.1,3.5,1.4,0.3,17.0,"'setosa'",0.0],[5.7,3.8,1.7,0.3,18.0,"'setosa'",0.0],[5.1,3.8,1.5,0.3,19.0,"'setosa'",0.0],[5.4,3.4,1.7,0.2,20.0,"'setosa'",0.0],[5.1,3.7,1.5,0.4,21.0,"'setosa'",0.0],[4.6,3.6,1.0,0.2,22.0,"'setosa'",0.0],[5.1,3.3,1.7,0.5,23.0,"'setosa'",0.0],[4.8,3.4,1.9,0.2,24.0,"'setosa'",0.0],[5.0,3.0,1.6,0.2,25.0,"'setosa'",0.0],[5.0,3.4,1.6,0.4,26.0,"'setosa'",0.0],[5.2,3.5,1.5,0.2,27.0,"'setosa'",0.0],[5.2,3.4,1.4,0.2,28.0,"'setosa'",0.0],[4.7,3.2,1.6,0.2,29.0,"'setosa'",0.0],[4.8,3.1,1.6,0.2,30.0,"'setosa'",0.0],[5.4,3.4,1.5,0.4,31.0,"'setosa'",0.0],[5.2,4.1,1.5,0.1,32.0,"'setosa'",0.0],[5.5,4.2,1.4,0.2,33.0,"'setosa'",0.0],[4.9,3.1,1.5,0.2,34.0,"'setosa'",0.0],[5.0,3.2,1.2,0.2,35.0,"'setosa'",0.0],[5.5,3.5,1.3,0.2,36.0,"'setosa'",0.0],[4.9,3.6,1.4,0.1,37.0,"'setosa'",0.0],[4.4,3.0,1.3,0.2,38.0,"'setosa'",0.0],[5.1,3.4,1.5,0.2,39.0,"'setosa'",0.0],[5.0,3.5,1.3,0.3,40.0,"'setosa'",0.0],[4.5,2.3,1.3,0.3,41.0,"'setosa'",0.0],[4.4,3.2,1.3,0.2,42.0,"'setosa'",0.0],[5.0,3.5,1.6,0.6,43.0,"'setosa'",0.0],[5.1,3.8,1.9,0.4,44.0,"'setosa'",0.0],[4.8,3.0,1.4,0.3,45.0,"'setosa'",0.0],[5.1,3.8,1.6,0.2,46.0,"'setosa'",0.0],[4.6,3.2,1.4,0.2,47.0,"'setosa'",0.0],[5.3,3.7,1.5,0.2,48.0,"'setosa'",0.0],[5.0,3.3,1.4,0.2,49.0,"'setosa'",0.0],[7.0,3.2,4.7,1.4,50.0,"'versicolor'",1.0],[6.4,3.2,4.5,1.5,51.0,"'versicolor'",1.0],[6.9,3.1,4.9,1.5,52.0,"'versicolor'",1.0],[5.5,2.3,4.0,1.3,53.0,"'versicolor'",1.0],[6.5,2.8,4.6,1.5,54.0,"'versicolor'",1.0],[5.7,2.8,4.5,1.3,55.0,"'versicolor'",1.0],[6.3,3.3,4.7,1.6,56.0,"'versicolor'",1.0],[4.9,2.4,3.3,1.0,57.0,"'versicolor'",1.0],[6.6,2.9,4.6,1.3,58.0,"'versicolor'",1.0],[5.2,2.7,3.9,1.4,59.0,"'versicolor'",1.0],[5.0,2.0,3.5,1.0,60.0,"'versicolor'",1.0],[5.9,3.0,4.2,1.5,61.0,"'versicolor'",1.0],[6.0,2.2,4.0,1.0,62.0,"'versicolor'",1.0],[6.1,2.9,4.7,1.4,63.0,"'versicolor'",1.0],[5.6,2.9,3.6,1.3,64.0,"'versicolor'",1.0],[6.7,3.1,4.4,1.4,65.0,"'versicolor'",1.0],[5.6,3.0,4.5,1.5,66.0,"'versicolor'",1.0],[5.8,2.7,4.1,1.0,67.0,"'versicolor'",1.0],[6.2,2.2,4.5,1.5,68.0,"'versicolor'",1.0],[5.6,2.5,3.9,1.1,69.0,"'versicolor'",1.0],[5.9,3.2,4.8,1.8,70.0,"'versicolor'",2.0],[6.1,2.8,4.0,1.3,71.0,"'versicolor'",1.0],[6.3,2.5,4.9,1.5,72.0,"'versicolor'",1.0],[6.1,2.8,4.7,1.2,73.0,"'versicolor'",1.0],[6.4,2.9,4.3,1.3,74.0,"'versicolor'",1.0],[6.6,3.0,4.4,1.4,75.0,"'versicolor'",1.0],[6.8,2.8,4.8,1.4,76.0,"'versicolor'",1.0],[6.7,3.0,5.0,1.7,8.589934592E9,"'versicolor'",2.0],[6.0,2.9,4.5,1.5,8.589934593E9,"'versicolor'",1.0],[5.7,2.6,3.5,1.0,8.589934594E9,"'versicolor'",1.0],[5.5,2.4,3.8,1.1,8.589934595E9,"'versicolor'",1.0],[5.5,2.4,3.7,1.0,8.589934596E9,"'versicolor'",1.0],[5.8,2.7,3.9,1.2,8.589934597E9,"'versicolor'",1.0],[6.0,2.7,5.1,1.6,8.589934598E9,"'versicolor'",1.0],[5.4,3.0,4.5,1.5,8.589934599E9,"'versicolor'",1.0],[6.0,3.4,4.5,1.6,8.5899346E9,"'versicolor'",1.0],[6.7,3.1,4.7,1.5,8.589934601E9,"'versicolor'",1.0],[6.3,2.3,4.4,1.3,8.589934602E9,"'versicolor'",1.0],[5.6,3.0,4.1,1.3,8.589934603E9,"'versicolor'",1.0],[5.5,2.5,4.0,1.3,8.589934604E9,"'versicolor'",1.0],[5.5,2.6,4.4,1.2,8.589934605E9,"'versicolor'",1.0],[6.1,3.0,4.6,1.4,8.589934606E9,"'versicolor'",1.0],[5.8,2.6,4.0,1.2,8.589934607E9,"'versicolor'",1.0],[5.0,2.3,3.3,1.0,8.589934608E9,"'versicolor'",1.0],[5.6,2.7,4.2,1.3,8.589934609E9,"'versicolor'",1.0],[5.7,3.0,4.2,1.2,8.58993461E9,"'versicolor'",1.0],[5.7,2.9,4.2,1.3,8.589934611E9,"'versicolor'",1.0],[6.2,2.9,4.3,1.3,8.589934612E9,"'versicolor'",1.0],[5.1,2.5,3.0,1.1,8.589934613E9,"'versicolor'",1.0],[5.7,2.8,4.1,1.3,8.589934614E9,"'versicolor'",1.0],[6.3,3.3,6.0,2.5,8.589934615E9,"'virginica'",2.0],[5.8,2.7,5.1,1.9,8.589934616E9,"'virginica'",2.0],[7.1,3.0,5.9,2.1,8.589934617E9,"'virginica'",2.0],[6.3,2.9,5.6,1.8,8.589934618E9,"'virginica'",2.0],[6.5,3.0,5.8,2.2,8.589934619E9,"'virginica'",2.0],[7.6,3.0,6.6,2.1,8.58993462E9,"'virginica'",2.0],[4.9,2.5,4.5,1.7,8.589934621E9,"'virginica'",1.0],[7.3,2.9,6.3,1.8,8.589934622E9,"'virginica'",2.0],[6.7,2.5,5.8,1.8,8.589934623E9,"'virginica'",2.0],[7.2,3.6,6.1,2.5,8.589934624E9,"'virginica'",2.0],[6.5,3.2,5.1,2.0,8.589934625E9,"'virginica'",2.0],[6.4,2.7,5.3,1.9,8.589934626E9,"'virginica'",2.0],[6.8,3.0,5.5,2.1,8.589934627E9,"'virginica'",2.0],[5.7,2.5,5.0,2.0,8.589934628E9,"'virginica'",2.0],[5.8,2.8,5.1,2.4,8.589934629E9,"'virginica'",2.0],[6.4,3.2,5.3,2.3,8.58993463E9,"'virginica'",2.0],[6.5,3.0,5.5,1.8,8.589934631E9,"'virginica'",2.0],[7.7,3.8,6.7,2.2,8.589934632E9,"'virginica'",2.0],[7.7,2.6,6.9,2.3,8.589934633E9,"'virginica'",2.0],[6.0,2.2,5.0,1.5,8.589934634E9,"'virginica'",1.0],[6.9,3.2,5.7,2.3,8.589934635E9,"'virginica'",2.0],[5.6,2.8,4.9,2.0,8.589934636E9,"'virginica'",2.0],[7.7,2.8,6.7,2.0,8.589934637E9,"'virginica'",2.0],[6.3,2.7,4.9,1.8,8.589934638E9,"'virginica'",2.0],[6.7,3.3,5.7,2.1,8.589934639E9,"'virginica'",2.0],[7.2,3.2,6.0,1.8,8.58993464E9,"'virginica'",2.0],[6.2,2.8,4.8,1.8,8.589934641E9,"'virginica'",2.0],[6.1,3.0,4.9,1.8,8.589934642E9,"'virginica'",2.0],[6.4,2.8,5.6,2.1,8.589934643E9,"'virginica'",2.0],[7.2,3.0,5.8,1.6,8.589934644E9,"'virginica'",2.0],[7.4,2.8,6.1,1.9,8.589934645E9,"'virginica'",2.0],[7.9,3.8,6.4,2.0,8.589934646E9,"'virginica'",2.0],[6.4,2.8,5.6,2.2,8.589934647E9,"'virginica'",2.0],[6.3,2.8,5.1,1.5,8.589934648E9,"'virginica'",1.0],[6.1,2.6,5.6,1.4,8.589934649E9,"'virginica'",1.0],[7.7,3.0,6.1,2.3,8.58993465E9,"'virginica'",2.0],[6.3,3.4,5.6,2.4,8.589934651E9,"'virginica'",2.0],[6.4,3.1,5.5,1.8,8.589934652E9,"'virginica'",2.0],[6.0,3.0,4.8,1.8,8.589934653E9,"'virginica'",2.0],[6.9,3.1,5.4,2.1,8.589934654E9,"'virginica'",2.0],[6.7,3.1,5.6,2.4,8.589934655E9,"'virginica'",2.0],[6.9,3.1,5.1,2.3,8.589934656E9,"'virginica'",2.0],[5.8,2.7,5.1,1.9,8.589934657E9,"'virginica'",2.0],[6.8,3.2,5.9,2.3,8.589934658E9,"'virginica'",2.0],[6.7,3.3,5.7,2.5,8.589934659E9,"'virginica'",2.0],[6.7,3.0,5.2,2.3,8.58993466E9,"'virginica'",2.0],[6.3,2.5,5.0,1.9,8.589934661E9,"'virginica'",2.0],[6.5,3.0,5.2,2.0,8.589934662E9,"'virginica'",2.0],[6.2,3.4,5.4,2.3,8.589934663E9,"'virginica'",2.0],[5.9,3.0,5.1,1.8,8.589934664E9,"'virginica'",2.0]],"arguments":{},"addedWidgets":{},"removedWidgets":[],"schema":[{"name":"sepalLength","type":"\"double\""},{"name":"sepalWidth","type":"\"double\""},{"name":"petalLength","type":"\"double\""},{"name":"petalWidth","type":"\"double\""},{"name":"id","type":"\"long\""},{"name":"specie","type":"\"string\""},{"name":"label","type":"\"integer\""}],"overflow":false,"aggData":[],"aggSchema":[],"aggOverflow":false,"aggSeriesLimitReached":false,"aggError":"","aggType":"","plotOptions":null,"isJsonSchema":true,"dbfsResultPath":null},"errorSummary":"Command skipped","error":null,"startTime":1.465536416392E12,"submitTime":1.465536404018E12,"finishTime":1.465536416617E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"823ad644-a7fd-470f-99b0-2a7586479a64"},{"version":"CommandV1","origId":124239,"guid":"d3123902-46fa-4ece-8fb1-2848a678309a","subtype":"command","commandType":"auto","position":4.0,"command":"%py\n#do plot with true label\nimport numpy as np\nfrom mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt\n\ndf = sqlContext.table(\"fisheriris\")\ny=df.collect()\nfea1 = np.asarray(map(lambda x: x[0], y))\nfea2 = np.asarray(map(lambda x: x[1], y))\nfea3 = np.asarray(map(lambda x: x[2], y))\ntl= np.arange(150)\ntl[range(0,50)]=0\ntl[range(50,100)]=1\ntl[range(100,150)]=2\n\nfig2 = plt.figure(figsize=(20,10))\nax = fig2.add_subplot(121, projection='3d')\nfigpars=[('r', 'o'), ('m', 'x'), ('b','^')] #color and marker\nsymbol=['setosa','versicolor','virginica']\nfor i in range(0,3):\n xs = fea1[tl==i]\n ys = fea2[tl==i]\n zs = fea3[tl==i]\n ax.scatter(xs, ys, zs, c=figpars[i][0], marker=figpars[i][1])\n plt.legend(symbol)\n plt.hold(True)\n\nplt.hold(False)\nfor ii in xrange(60,160,30):\n ax.view_init(elev=10., azim=ii)\nplt.axis('tight')\nplt.title('Iris flower class')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\n\n#do plot with kmean\ndfkm = sqlContext.table(\"HOSpectralTable\")\nyy=dfkm.collect()\nfea1 = np.asarray(map(lambda x: x[0], yy))\nfea2 = np.asarray(map(lambda x: x[1], yy))\nfea3 = np.asarray(map(lambda x: x[2], yy))\ntl = np.asarray(map(lambda x: x[6], yy))\n\nax = fig2.add_subplot(122, projection='3d')\n#figpars=[('r', 'o'), ('m', 'x'), ('b','^')] #color and marker\nsymbol=['cluster_0','cluster_1','cluster_2']\nli=[2,1,0]#This is used just to match the marker of two figure below more comparable\nfor i in range(0,3):\n xs = fea1[tl==li[i]]\n ys = fea2[tl==li[i]]\n zs = fea3[tl==li[i]]\n ax.scatter(xs, ys, zs, c=figpars[i][0], marker=figpars[i][1])\n plt.legend(symbol)\n plt.hold(True)\n\nplt.hold(False)\nfor ii in xrange(60,160,30):\n ax.view_init(elev=10., azim=ii)\nplt.axis('tight')\nax.set_xlabel('X Label')\nax.set_ylabel('Y Label')\nax.set_zlabel('Z Label')\nplt.title('HOSpectral-cluster')","commandVersion":0,"state":"finished","results":{"type":"html","data":"<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">7</span><span class=\"ansired\">]: </span><matplotlib.text.Text at 0x7f9ccc658a10>\n</div>","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536416625E12,"submitTime":1.465536404025E12,"finishTime":1.465536417333E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"24b4f66f-021e-4650-b971-f255ebfddfb0"},{"version":"CommandV1","origId":124240,"guid":"4bfeb7b0-1b41-449c-89de-3b888314f014","subtype":"command","commandType":"auto","position":5.0,"command":"%py\ndisplay(fig2)","commandVersion":0,"state":"finished","results":{"type":"image","data":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAB9AAAAPoCAYAAACGXmWqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xu4llP++PH3enaHXToplKmhUCEjSVKREoohZMgXoyKnfJ19f2OGoZohhiENOQ5yPmXEYHIsaSq7qFChSeWQDqikg93e6/fH/extH559yk4N79d1rWu373W8n4fr2uv+3GutEGNEkiRJkiRJkiRJkqSfu9SWHoAkSZIkSZIkSZIkSVsDA+iSJEmSJEmSJEmSJGEAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSYABdEmSJEmSJEmSJEmSAAPokiRJkiRJkiRJkiQBBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZIkSRJgAF2SJEmSJEmSJEmSJMAAuiRJkiRJkiRJkiRJgAF0SZIkSZIkSZIkSZIAA+iSJEmSJEmSJEmSJAEG0CVJkiRJkiRJkiRJAgygS5IkSZIkSZIkSZIEGECXJEmSJEmSJEmSJAkwgC5JkiRJkiRJkiRJEmAAXZIkSZIkSZIkSZIkwAC6JEmSJEmSJEmSJEmAAXRJkiRJkiRJkiRJkgAD6JIkSZIkSZIkSZIkAQbQJUmSJEmSJEmSJEkCDKBLkiRJkiRJkiRJkgQYQJckSZIkSZIkSZIkCTCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSZIkSZIkSZIkSQDU2NIDkCRJkrR1CCHsBGy3pcchVWBFjHHxlh6EJEmSVJTzKW1FnDNJP5ABdEmSJEmEEHZKpVIf5OfnZ2/psUjlSaVS60MIbX0gJEmSpK2F8yltTZwzST+cAXRJkiRJANvl5+dnP/TQQ+yxxx5beixSRnPnzuXUU0/NJlnZ48MgSZIkbS2cT2mr4JxJqh4G0CVJkiQV2mOPPdh333239DAkSZIk6b+O8ylJ+mlIbekBSJIkSZIkSZIkSZK0NTCALkmSJEmSJEmSJEkSBtAlSZIkSZIkSZIkSQIMoEuSJEmSJEmSJEmSBBhAlyRJkiRJkiRJkiQJMIAuSZIkSZIkSZKkH8miRYtIpVKcfvrpW3ookpSRAXRJkiRJ2szGjBlDKpXigQce2NJDkSRJkqSfnaFDh5JKpXjjjTe29FAyys/P5+abb6Z9+/bUrVuXJk2a8Otf/5opU6Zs6aFJP0sG0CVJkiTpRxBC2NJDkCRJkqSfpRDCVj0n69+/P5deeim5ubmcf/759OvXj0mTJtG9e3eee+65LT086WenxpYegCRJkiT91MUYt/QQJEmSJOlna2uekz366KOMHTuWAw88kFdeeYVatWoBcM4559CtWzfOPPNM5s+fT7169bbwSKWfD1egS5IkSfrRfPjhhwwZMoTO++5L36OO2irfpP/HP/5Bz5492XHHHcnOzmbHHXfkoIMOYvTo0cXKff311/z+979nzz33pG7dujRq1IhDDz2Ul19+uVi5nj17Fp7tN3DgQFKpFKlUiqysLBYvXlxYbtWqVVx++eW0bduWOnXq0LhxY3r37s0rr7yScZz33nsvXbt2ZYcddqBOnTo0b96cww47jMcff7xYuQkTJnDWWWfRrl07GjZsSN26ddlrr70YOnQo69evr46PTJIkSdKPYMWKFVx99dV03X9/evXowV133UVubu6WHlYpOTk59O/fnxYtWpCdnc0vfvELevfuzZNPPlluvR49epBKZQ5blXUs1syZM+nfvz8tW7YkOzub7bbbjvbt23PRRReRl5cHQKtWrRg+fHixPgrmZEWtW7eOESNG0KFDB+rVq0f9+vXp2rUrjz32WKnxTJw4kVQqxfDhw5k2bRp9+vShcePGpeZ5lTF69GhCCFxzzTWFwXOAjh070r9/f5YvX85TTz1VpTYl/TCuQJckSZK0SWKMrF+/ntq1a5f5kKOoadOm0atnT+rn5tJn40bmzp5N3+ef549//GPhw4wt7fbbb+e8885jxx135JhjjmG77bZj2bJlzJ49mzFjxjBkyBAAFi9ezMEHH8zixYvp3r07Rx55JGvWrOGf//wnffr04a677uKMM84AYNCgQWy77baMGzeOY489ln322QdIthBs1KgRACtXrqRLly58+OGH7L///hx//PGsWLGCJ554gt69e3PbbbdxzjnnFI7zd7/7HTfccAO77LIL/fv3p2HDhixZsoScnBzGjh1L//79C8tef/31fPDBB3Tt2pWjjjqKdevWMXnyZIYPH86ECRN47bXXKvX9SZIkSapeGzZsICsrixo1Kg7VLFmyhG6dO7Pss884Oj+flSFwzsSJPDduHM88+2ypYPCWcvfddzNkyBBq1KhB3759ad26NcuWLSMnJ4fbb7+dE044ocy6FW2zXjJv1qxZdOnShaysLPr27UurVq1YvXo18+fP54477uDaa6+lbt26XHzxxTzzzDNMnDiRgQMH0rJly1LtrVq1ip49ezJr1iw6duzIGWecQX5+PuPHj+fkk09mzpw5GeetkydP5pprrqF79+4MHjyYZcuWFQuCV2TDhg1MnTqVunXrcuCBB5bKP+KII3jwwQd5/fXXGThwYKXblfQDxRhNJpPJZDKZTCbTzzwB+wJxxowZsTIeeeSRuGebNhGIDevVi5dcckn89ttvy63Teb/94n6pVFwDMabTUIghhLhgwYIK+1y3bl2cM2dOXLZsWaXGuCn23XffmJ2dHVesWFEq78svvyz898EHHxyzsrLi2LFji5VZtWpV3GeffWLdunXj0qVLC6/ff//9MZVKxTFjxmTs98wzz4whhPi///u/xa5/+OGHsUGDBrF27dpx4cKFhdcbN24cW7RoEdevX1/uOGOM8eOPP87Y5xVXXBFTqVR8/PHHM+ZvjWbMmBGBCOwbt4L/b0wmk8lkMplMphirPp968803Y/du3SIQa9aoEf/npJPip59+Wm6d8847L25Xo0ZcWGQ+9Xzyt3F86qmnKuxz48aN8YMPPoiLFy+u1Bg3xZw5c2LNmjVjkyZN4rx580rlF9zjwoULYwghDho0qFh+jx49YiqVyth2pjnVJZdcElOpVHz22WdLlV+5cmWx34cOHRpTqVScOHFixvYHDBgQU6lUvPnmm4td37BhQ+zTp0/MysqKM2fOLLw+YcKEGEKIqVQq3n333RnbrIz3338/hhDi3nvvnTF/+vTpMYQQDzjggEq155zJZKqe5DIDSZIkSVUyZswYTj75ZHb76CPuB85Zs4Y7Ro7kN8cdR4yZz5VbunQp06ZP55L8fLYpcv3/gJpQ7lbuMUZGjBjBL5o2Zc8996RZ06Ycf9xxLF++vBrvKhFCoEaNGhlXbzRu3BiA2bNn88Ybb/Cb3/yGfv36FSvToEEDhg0bxvr16xk7dmyl+szNzeXhhx+mfv36/PnPfy6W17p1ay644AJyc3N58MEHi42zVq1aGVdnFIyzQMHqipIuvvhiYoy89NJLlRqnJEmSpB9u+vTp9OrZk7VTpnAH8KeNG5n45JN079qV1atXl1nv2bFj+e3Gjexc5NqRQIesLMaNG1dun08++SS7tWxJ27Zt2Wmnnei6//7Mnj27Wu6nqNGjR5OXl8dVV11F27ZtS+U3b968WvsrmA/VqVOnVF7Dhg0r3c5XX33Fww8/TKdOnbjooouK5dWqVYvrr7+e/Px8HnnkkVJ199lnHwYPHlzFkX9v1apV5Y634PrKlSs3uQ9JVecW7pIkSZIqLS8vj6uvuIL+wKMxUhC+3T8/n+NfeompU6fSpUuXau3zxhtv5A9/+AMXAP2AuTFy9XPPceThhzNtxoxq3X78lFNO4bLLLqNdu3acdNJJdO/enW7durHddtsVlpkyZQqQPMAYNmxYqTaWLVtGjJF58+ZVqs8PPviAdevWcdBBB2V8aNKrVy+uueYa3n777WLjvPXWW2nXrh0nnngi3bt3p0uXLjRo0KBU/bVr1zJy5EieeeYZPvzwQ7755pvCFx1CCHz22WeVGqckSZKkH+7aP/+ZXfLzeTM/n9rpa7/Jy2P3Tz5hzJgxnH/++dXa3yuvvEL//v3pGyN3ACuBa95+m149evD+vHnssMMO1dbXtGnTAOjTp0+1tVmek046iVtuuYVjjjmGE044gV69etGtWzd22WWXKrWTk5NTeF56pjned999B5Bxjrf//vtvwsglbe0MoEuSJEmqtM8//5xFn33G34Cia5+PBbZJpZg8eXLGAHrTpk3pvN9+3PT22/Qtsgr9BiAXOProozP2l5uby43XXce5wC3pawcDu+fl0XPmTF599VUOO+ywarq7ZFX29ttvz+jRoxk1ahQjR44EoGfPntxwww106NCBL7/8EoCXX36Zl19+OWM7IQS+/fbbSvVZsOKgWbNmGfN33HHHYuUARo4cya677sp9993Hddddx4gRI6hRowa//vWvuemmm2jVqhUAGzdupGfPnuTk5PCrX/2Kk046ie23356aNWsCMHToUDZs2FCpcUqSJEn64f49aRJn5+UVBs8BdgW6hMDkyZPLDKD3Pf54HrjjDi7Myytchf4C8E5eHlccc0yZ/f1lxAg6pVI8nZdXuCVxz7w8Wq5axT333MMf/vCHarirRMEq6epeaV6W/fbbjzfffJNrrrmGp556igcffJAYI7vvvjtDhw7lxBNPrFQ7BXO8nJwccnJyMpYpa45X1jyusgpeoi463yuq4HqjRo1+UD+SqsYt3CVJkiRVWoMGDchKpVhU4vpSYF2MpbYPL+qWW29lbu3a7FajBoOAzllZDAWuvPLKwoBvSUuWLGHZV19RMrx+MFA/K4uZM2du8r2U5dRTT+Xf//43X375Jc8//zyDBw9mwoQJ9O7dmy+//LLwAcctt9xCXl5ememee+6pVH8F7X3xxRcZ85csWVKsHCQPby644ALeeecdli5dytNPP02/fv0YN24cRxxxBBs3bgRg3Lhx5OTkMGjQIGbNmsUdd9zBn/70J6666irOOuusTf6MJEmSJG2axo0bs7DEtXxgcSpV7nzqiiuuoMEvfkG7rCz+B+gTAkcBRx15JMcee2yZ9Wa98w6/LhI8B9gBOACYNWvWpt5GRgVB3k3d5apgd7H8/PxSeWVtYd65c2eeffZZvv76ayZPnsxVV13F0qVLOfnkk5kwYUKl+i2Ya1188cXlzvFeeeWVUnUzHatVFbvuuitZWVksWLAg431/9NFHALRp0+YH9SOpagygS5IkSaq0hg0bctyxx3JNVhYFG4p/BZwbAnXr1OH4448vs27nzp15e+ZM+p11FnM6dKDZEUfw7LPPMnz48DLrNGnShNo1a1Lysc4C4Ju8PFq0aPFDb6lMDRo0oE+fPtx5550MHDiQL7/8kkmTJnHAAQcAMGnSpEq3lZWVRYyxcFvAotq2bUvdunWZNWtWxjMPX3vtNQA6duyYse3tttuOY489lscee4xDDjmEjz76iPfeew+A+fPnE0IodVY7UOmHSZIkSZKqz4DBg3k0leJpIAIbgCuBRRs3MmDAgDLr7bjjjrz19ttcesUVLOrUidzu3bnzrrt4+plnyMrKKrNe8xYtmFUiyPsdMCeVqvaV4gVzpfHjx29S/W233RaATz75pFReWSvDC9SsWZMDDjiAoUOHcsstt5Cfn88zzzxTmF/wGWWak+2///6kUqkqzfGqS+3atenatStr167N2P8LL7xACIFevXr96GOTfs4MoEuSJEmqkr/ddhtNdtuNjkCrmjX5RSrFS7Vq8ejjj2c8w7uoNm3acNtttzHt7bcZ99xzZW7dXmCbbbbht6edxrVZWTwN5AFzgd+mUuzQpEm5Ky02RVlB5aVLlwKQnZ1Nx44dOeigg3j66ae57777MpZ/7733WL58eeHvTZo0ATI/CKpZsyannHIKq1ev5o9//GOxvP/85z+MGjWKWrVqceqppwLJ+Xv//ve/S7WTm5tbuPVgdnY2AC1btiTGyOuvv16s7IIFC7j88st/8GoJSZIkSVVz8cUXc8SRR3I80LxGDZpmZTECGDFiBJ07dy637nbbbcewYcP491tv8eqECZx55pmFxzOV5ezzzuPpGLkJWAt8AQwGlufnc8YZZ1TPTaWde+65ZGVlMXz4cD744INS+RWtTO/cuTMxRu6+++5i11999VUee+yxUuWnTJnC+vXrS10v2N2rYF4EyZwsxphxTrb99ttzyimnMH36dP785z9nXAm+YMECFi5cWO74N9W5555LjJErr7yy2BFbOTk5PPHEE+ywww4ZX4qWtPl4BrokSZKkKmnWrBlvz57NuHHjmD59Os2aNePkk0+madOmm6W/m26+mcULF3L8q6+SItnecMcmTXjun/+kTp061drXcccdR7169TjggAMKg8+TJk0iJyeH/fbbj0MPPRSARx55hF69ejF48GBGjRpF586dadSoEZ9++imzZ8/m/fffZ8qUKWy//fYAdOnShbp16zJy5EhWrFhR+FldcMEF1K9fn+uuu45JkyZx66238tZbb9GzZ0+WL1/Ok08+yZo1a7jtttvYeefkpMN169Zx4IEHsttuu9GxY0d23nln1q9fz8svv8y8efM4+uij2X333YHkbPnddtuNm2++mXfffZcOHTqwaNEinn/+eY466qiMD6EkSZIkbT61atXimWefZcKECbz88svUqVOHE088kbZt226W/s4880zef+89LrvtNi6LkQjUqV2bMffcQ7t27aq1rz322IPRo0dz7rnnss8++9C3b19at27NihUryMnJoVGjRrz66qtl1h80aBA33ngjI0aMYObMmey55558+OGH/Otf/6Jfv3489dRTxcr/5S9/4bXXXuOggw6iVatW1KtXj/fff58XX3yRxo0bFzu2qmfPnqRSKS6//HJmz55duNr9iiuuAODWW29l/vz5XH311Tz44IMceOCBNG3alM8//5y5c+cyffp0Hn30UVq2bFmtnxnASSedxNNPP83YsWPp0KEDRx99NCtWrOCJJ54gPz+fu+++m3r16lV7v5LKEWM0mUwmk8lkMplMP/ME7AvEGTNmxK3VW2+9FUePHh2ffvrpuH79+s3Sx5133hn79esXd91117jNNtvEJk2axH333TfeeOONcc2aNcXKrlmzJo4YMSLut99+sX79+rFu3bpxl112iUcddVS855574tq1a4uVHz9+fOzatWusX79+TKVSMZVKxUWLFhXmr1q1Kl5++eWxTZs2MTs7O2677baxd+/e8ZVXXinWTm5ubrzhhhvikUceGXfeeedYp06duMMOO8QuXbrEu+66K+bm5hYr/+mnn8ZTTz01tmjRItatWzfutdde8cYbb4wbN26MqVQqHnLIIdX8KW4+M2bMiCQ7Xe4bt4L/b0wmk8lkMplMphj/O+ZT8+fPj3feeWd84IEH4ldffbVZ+5o6dWr8zW9+E5s2bRpr164dmzdvHo844og4duzYGGOMCxcujKlUKp5++uml6s6dOzceddRRsUGDBrF+/fqxZ8+ecdKkSfH++++PqVQqjhkzprDsyy+/HE8//fTYrl272KhRo1ivXr24++67x4suuiguXry4VNsPP/xw7NChQ6xbt25MpVIxKyurWH5ubm687bbbYrdu3WKjRo1idnZ23HnnneOhhx4aR40aVexzmzBhQkylUnH48OHV8pnl5eXFkSNHxr333jvWrVs3Nm7cOB511FFx6tSpVWrHOZPJVD0pxBi3VOxekiRJ0lYihLAvMGPGjBnsu+++W3o4UkZvv/12wVnwHWOMb2/p8UiSJEngfEpbD+dMUvXwDHRJkiRJkiRJkiRJkjCALkmSJEmSJEmSJEkSADW29AAkSZIkSZIkSZKkn4KRI0eyatWqCsv16NGDgw8++EcYkaSqMoAuSZIkSZIkSZIkVYNbbrmFxYsXV1guhGAAXdpKGUCXJEmSJEmSJEmSqsHHH3+8pYcg6QfyDHRJkiRJkiRJkiRJkjCALkmSJEmSJEmSJEkSYABdkiRJkiRJkiRJkiTAALokSfqRhBAmhBDyN1PbO4QQ7gshLA4hbAwh5IUQGoQQBoYQ8kMIp22Ofv/bhRB2Tn8+927psUiSJElSVW3OeWa6/fwQwmubq31JkrR1MoAuSZKqJP0AIW8TqkZgcz3YuB/4LTAR+BMwFFif7jNupj4lSZIk/cRUZr4TQliYLrdTGfmtQwi3hRDmhhC+CSGsCSHMS19rU067tUMIl4UQpoYQVoYQNoQQPg8hTA8h/C2E0P2H3t/mFEIYsAVeYP5Jzfm20GcoSZJKqLGlByBJkn42fgvUre5GQwg1gcOBl2OMvy2RV93dSZIkSVKZQdsQwgXAX4FA8oLvc+myHYGzgbNCCJfEGP9Wot42wCRgH2AJ8BTwBVAvfe0coCHwxma4n+r0kwlmb0F+hpIkbWEG0CVJ0o8ixvjpZmp6R5JddZZspvYlSZIkqUIhhAHASGAFcFyMcXKJ/G7AOGBkCOHrGONDRbIvJgmU/wvoG2PcWKJuU2C3zTn+auAbzD+cn6EkSVsBt3CXJEk/WNGztEMIbUMIT4UQlqXPIu+eLpPxbLoQwukhhH+ny68LIXwWQng5hNC/Ev1+DCwkeUO/4LzzSp3pHULoGEJ4OoSwNISwPr0N4+gQwo4lyl2bbrNXievXpK9/lKHtL0IICzNc7x1CeCGEsDzd5/wQwl9CCA0zlF0YQliQPsv9lvTv34UQrqro3tL1O4UQHg8hfJru6/MQwvgQwgmVqNs6hHBdCCEn/b0UfD53hRB+WUadSn2PIYR90uNamG53RQhhVghhZAghqzL3JkmSJG1tQgj1SYLnETilZPAcIH3tFJIg6cj0qvMCXdJ17ygZPE/XXZohID80PSfpnt76++0Qwtr0HOfv6aB7prFuG0IYEUKYky6/MoTwSgjhsHLur38I4dUQwpfpv/c/DiE8EkLYN53/OlAwD7u/yNwsL6S3ui8x3tNCCG+FZHv7BUX6GZieT/4nPbZVIYQ3Qwi/LT2qTZeet96bvo/16c/sjRDCOZWoW3B/pbbwDyEcnM67qsT1XUMI96TngOtCCF+lP/87QgjbpstU+Bmmy2WFEIaEEKakP59v09/9eSEU34YtVGKuLkmSSnMFuiRJqk6tganAPOBBku0Gv0nnldrmMIRwPfB/wALgcWAVyYryTsDx6WvluRloCVwEzASeSV+fWV6lEMIxwBMkZ7I/BSwG9iPZFrFvCOHAGOPCdPFXgcuBQ9L/LnBI+n52CSHsFGNcnG67HbAD3z/4KOjzauBqktUo/wSWAXsDlwFHhBC6xhi/KVIlArWB14BGJCtR1pC8MFCuEMKZwGhgI/As8FF6TJ2Ac4EnK2iiH3AW8DowGfgO+BVwBnB0CKFjjPHzIv1V6nsMIbQHpgB56XF9DDQgWUlzDvAHYG1F9ycJFi1aRKtWrRg4cCD33lvhO0PVYsyYMQwaNIj777+f007zWE6l0IgqAAAgAElEQVRJkkr4DckW62/FGF8uq1CMcXwIYTrJlu6/Acaks1akf5Z5Rnqm5tLpEuAwkr+7XwQOBAYBB4cQOscYvyyokA7ETgR2ItkO/gWSedtRwL9CCGfFGP9etJMQwv3AacByYGz65y+BniRzv7eB+4CvgWNI5mUFc7IIrCwx3v8DepFsb/8qyXynwGjgvfQYlwDbAb8GxoQQ2sYYr6zC55NRCOHXJPPBWiTzrEeAbYH26bHdUUETVTp3PSQvaecA25B83k8C2UArkhcqRpF8dhV+hiGEGiTzycOBucDDwHqS7+JvQGeS76qk8ubqkiSpBAPokiSpOnUFro0x/rGS5QcDnwHtYowbimaEEBpXVDnGOCqEsDPpAHqMcXhFddKrPO4lWfXRI8Y4rUje74ARwJ1A7/TlycAGkgc8V6TLNSAJuL9M8qCqF8nDDtL/jiTB54J2e5IEzycBv44xrimSdxpwPzCM5MFXUc1IHh4dFGNcV9G9pdvbA7iNJIjdLcb4QYn85pVo5gHgphhjbom6vYHngSuBIUWyKvs9nkbykOrYGONzJco1jDEaPJeqIIRAiUVGP0qfkiT9HKRfgM2YRfGAb4EDSeYBZQbPi3iZZD7Rje8D6I8BvwX+HELYhSQQPqPoi6vljKcPsH+McXaR8d9EMk+6DjizSPkHSILfJ8QYny5SvgFJ0HpUCOG5GOOy9PWzSP6OnwocXmIuE0he1CXG+ED692OAZ2KMD5Qz3h7AAUXHW0S7GOPHxSqEUBMYD/xfCGF0JT6TMoUQmpAEzLNI5oMlV/VXZr5UVQUvV1wUY/xbif7qkLzYXdnP8EqS4PnNwGUxxphuJwB3A4NCCE/FGJ8tUa+qc3Vps9oSLwRLUlW4hbskSapOS4EKg9hFRJLVzaXe3o8xflVdgyrhWJLVBY8WDZ6n/ZVkhfehIb1VeYxxPcmq6X1Dsi0jJA98UunyX5EEzQscmv5ZdLX6BST3eHbRB07p9h8gWVlwchnjvayywfO0ISQPg4aXDJ6n+/usogZijEtKBs/T18cDc0ge2BTLonLfY0F+qfuJMa6qaFySvte8eXPmzp3Ltddeu6WHIknST9VVZaQ/kgRDS2qW/vlJJdouKPOLggsxxheB84BvgbNJzkr/NCRHIz0QQuhSTnsPZAhGDyV5qfbkdACaEMLeQHfgqaLB83T/q0le+s0m2UWqwPkkf8efk2EuE2OMSytxvyXdVUbwnJLB8/S1XOBWksVgh2xCf0UNBOoDo8vYZr/C+dImiCQvDmSaB60r+RJyWdJB8v8FPqdI8DzdTgQuTf96SobqVZ2rSz85Q4cOJZVK8cYbb2zpoZQyb948rr76ao499lh23nlnUqkUWVlZ5OeXOglR0o/EFeiSJKk6zcoUeC3HwyQPAN4PITxBsoXglPTDm82lA8kDjAklM2KMG0MIb5Cs/OjA9w+2XgMOJgmcP0fy0GZDeryvp38nhJACDgI+iDF+UaTpA4BcoH8ZqzdrAduHELaNMX5d5Pr6GOO7Vby/zumf/6pivWJCCKcCA0i2MdyWJChfoOQDnsp+j48BFwLjQghPkrxkMDnGuABJVVKjRg3atKnKDq+SJKkqYoxZZeWFED4m2QI9Y9XKNF9Gn7eHEO4j2eWqK8mcpCtJQPTUEMLQDLtuRZK/v0u2tTqEMJMkYL4HMJvknHWARmWssN+BJNC7O0AIoS7QDviirID3JojAW2Vlpl9kLjhCayegTom65a4QDyE0BC6m9Gd8c3p+0jmd94PmS1X0LHAtMDqEcDjJDgSTY4xzqthOG6Ax8CFwVYa5ZUGQfvcMdas6V5d+crbEDl6VNX78eP70pz9Ro0YNWrduTXZ2Nhs2VOrdGkmbiSvQJUlSdfqi4iLFXETycGMNyUOSF4EVIYR/hBBaVffg0gpWi5Q11iUlykES6A18v9L8EJIA8YZ0XtMQwp4k2zA2pPjqc4AmQE3KXsWyB8lDnHol6i2r9F19r2A7yU1eORFCuJlka8c9SB4s3UiygmUosIgk4F9Upb7HGON0kq0tXyXZxvB+YH4IYU4I4cRNHa/+e6z9cC2r3yr+XsXGbzayYtyKMmr8uKZNm0YqleL4448vs0ybNm2oU6cOK1euLLw2fvx4jjzySLbffnuys7PZbbfd+H//7/+xalXpjRVatmzJLrvswurVq7nwwgtp2bIltWrVYvjw5Fn46tWrGTZsGHvttRcNGjSgfv36tGrVihNPPJF33nmnsJ1FixaRSqU4/fTTS/Wxbt06rr/+ejp16lTYxp577smFF17I8uXLi5VdsmQJQ4YMoVWrVtSuXZsddtiBfv36MX369Cp9djNmzKBfv340bdqU7OxsWrZsyZAhQ1iyZEmpsgMHDiSVSrFw4UJGjhzJ3nvvTd26dTnkkB+6oEySpC2qYH7xy0qUbZH+WWor8hjj+hjjczHG38cY+5AETP8XyAOuTq8iL6msVeAFYyqY2zRJ/zyMzPOSc0jmJduky/3guUUF4yomPXd4BziLZF52N/AnknnI/elitStouxGZ761RkXyo/nsqU4xxMdCJ5Pz4w0jOWH8vhLAohHBeFZoq+P5aU/buCHX4/vsrqqpzdeknp8imDVudI488kmnTpvHNN9/w/vvv07Rp0y09JOlnzwC6JEmqTlWajaS3/BsVY+wANAX6AU+TnPn2Yghhc+yWUxDRalZG/o4lykGyQmIN0CuEsD2wF98HyQuC64fy/fnnr2Xo8+sYY1Y5qUaMseR2j5syuyuI6m3S2X3p+zufZIVKmxjjaemHd8PTq11KvQJdle8xxjgtxtiXZFV7N5JtBJsCj4QQemzKmPXf4+MrP2bWYbNYPS0Jom/8ZiOzj5jNvIHzyP1yyy+I6dy5M23atOGFF17g66+/LpU/ZcoU5s+fT9++fWnUKHn2OmzYMI444gimT5/OUUcdxYUXXkjr1q258cYbOfDAA/nmm2+KtRFCYMOGDRxyyCE8//zz9OnThwsuuICWLVsC0Lt3b4YPH06jRo0466yzOO+88+jWrRuTJ09mypQpFd7DypUr6dKlC7///e9Zu3YtZ5xxBkOGDGGPPfbgvvvuY+7cuYVlP/74Yzp27Midd95J69atueyyy+jTpw8vvvgi3bp147nnnqvU5zZu3Di6du3Kiy++yOGHH86ll15K27ZtueOOO+jUqRMLFy4s9RmEEDj//PMZPnw47du358ILL6Rr166V6k+SpK3Um3w/L6jIoSR/65faQrykGOPGGOPtwKPpS5neOCsr0lIw51lV4ueFFcxNBqfL/aC5RTnKmudcSjJPOD3GeEiM8aIY49XpechLJJ9v+Q3HuCjGmMpwT4vTRarjngr2VM40X22U4Roxxg9ijP9DEgTfD/gdyf2MCiEMqGS/Bd/fPyr4/nbLNIRK9qGt2MZVG1nxXPGXj2N+ZOmjS7e64HBOTg79+/enRYsWZGdn84tf/ILevXvz5JNPlluvR48epFKZw1ZjxowhlUrxwAMPFLs+c+ZM+vfvT8uWLcnOzma77bajffv2XHTRReTl5QHQqlWrwpeWC/oo2Ca9qHXr1jFixAg6dOhAvXr1qF+/Pl27duWxxx4rNZ6JEyeSSqUYPnw406ZNo0+fPjRu3JisrCwWL15cqnx5WrduTadOnahdu6J3hCT9WNzCXZIkbRVijCuAZ4BnQgjbAT1JAtUzq7mrd0geVPQA7iuaEULIItmCHeDtImPLS2/tfgTJWeWRdAA9xjg/hPAJyUOwOiQPU14v0edU4MgQwh4xxrlsXlOBjkBvkq39qmoXkpcsX44xri2aEUJokc4vU2W/x/T2gVOBqSGE+SQr3o8lw9b62vos/8dyvlv6Hc3P+f6547dzv2XJ3UvY9YZdCVmZn222/XtbZh8xm1mHz2Kvp/fi46s/5tt3v6X9S+2p2aTmjzX8cg0YMIArr7ySRx99lCFDhhTLGzNmDCEEBgxInnG+/vrrDBs2jIMOOojnn3+eevW+30TigQceYODAgVx99dXcdNNNxdr54osv2GuvvZg0aRJ16ny/K+m7777LtGnTOO644xg7dmypsWVa0V7SkCFDePfddznvvPP429/+Vizv22+/LXyABHD22WezdOlSrr/+ei677LLC6+eccw4HH3wwgwYNYtGiRWyzTaZFTN+3efrppxNjZMKECXTu3Lkw7/rrr+f3v/89Z599NuPHjy9WL8bIzJkzmTlzJjvtVNYOuJIk/Vd5Cvgr0DmE0CvGWHJXKgBCCIeRbCO+Il2nsgreyiv5h1YgOW7qoRL9NAD2AdYDBXOQqemfB5GcKV6uGOPaEMJ7QLsQwt6V2MY9Lz2eMre/r8Cu6Z9PZ8jrQfUEgaeS7IbVmyQovykK3rT8JVDyOKpOlDPOGGM+yZz0nRDCFJLt948DxqSLlPcZziN5AeCAEEJWjDEvQxn9hH1+5+csuHwBu9+3O80GNCPmRz48+0OW/H0J2S2zadilYcWN/AjuvvtuhgwZQo0aNejbty+tW7dm2bJl5OTkcPvtt3PCCSeUWbeibdZL5s2aNYsuXbqQlZVF3759adWqFatXr2b+/PnccccdXHvttdStW5eLL76YZ555hokTJzJw4MDCF5iLtrdq1Sp69uzJrFmz6NixI2eccQb5+fmMHz+ek08+mTlz5hQG4YuaPHky11xzDd27d2fw4MEsW7aMWrVKbtwn6b9OjNFkMplMJpOp0okkQJxX4trO6ev3llPv9aL1SLYB75qhXE2SBwp5wO6VGE+ZfZOc4Z0HnFbk2jYkD6u+AzqXKH9Zuq1/ZWjrknTeFyQPLVJF8u4lWQ2wFpieoe4h6bpvAjtmyK+bYSwfAws24fvZI31vy4G2GfKbl/fZkaxeyQemlLjHeiRbsxf7/qvyPZKcuZidoWzB537dlv7v++ecgH2BOGPGjFiRjy79KL7O6/HT0Z/GGGNcM2dNfLPpm/Gtvd6K3339Xbl1c1fnxrfavxVf5/X4Oq/HVVNXVdjfJ3/7JP7n8v/E/Pz8wmtfv/F1fPe4d+PGtRsrrF8Vn3zySczKyor7779/sesbNmyI2267bWzWrFnMy8uLMcZ47LHHxlQqFefOnZuxrQ4dOsSmTZsWu9ayZcuYSqXi7NmzS5V/9913YwghnnLKKRWOc+HChTGEEAcNGlR4bdmyZTErKys2b948rlu3rtz6n376aQwhxFatWhXeT1GnnnpqTKVS8cEHHyy8dv/998dUKhXHjBlTeO2hhx6KIYR42mmnlWojNzc3tmrVKqZSqbh48eLC6wMHDoypVCqOGjWqwvssacaMGZHkofS+cSv4/8ZkMplMP72Uab6ToczH6b9zdypxfVC6/rIy/kbumv47PQ84pUTe2SXnBEXydk+3mQd0LHL96nR/64F9StS5OZ13d4nrE4FcYFAZfe0FbF/k98HpdiYD9UuUTQHNivx+RLrsVWW0fXX6HrqXkX97Ov+oEtd7p8ecV7JtSswzK/H9NiGZz60r4ztqXuL3fOC1EtdOSF9/qMT1XwGrS44z/Xd2gwx9/SbdzmNV+AyHpfNHlzG3agbsUeT3CufqpupJVZlPffPuN3H+ZfNjft7385vvvvouzjtrXsxdmVtmvfy8/Dhv8Lz4eng9fn7v54X/XnL/kgr7/LHMmTMn1qxZMzZp0iTOmzevVP6nnyZzyEzzmRhj7NGjR0ylUhnbzjQfueSSS2IqlYrPPvtsqfIrV64s9vvQoUNjKpWKEydOzNj+gAEDYiqVijfffHOx6xs2bIh9+vSJWVlZcebMmYXXJ0yYEEMIMZVKxbvvvjtjm5uqYN6Yaa5WEedMJlP1JFegS5KkLaUO8GZ69fEMkrO1s0nOhNsdeC7GOK8a+in2enKM8dsQwunAE8DEEMKTwGKSVduHk5xDeE6GdgpWkGwP/DMmKweK5g2kyMr0En2+FkL4HTAC+CiE8ALJQ7d6JA80DgYmAUdu4j0W7WtuCGEIycOnmSGEZ4GPgO1IVkOs5Puz3DPVXxpCeAzon67/EsmZiYeRPGSaCbQvUqUq3+P/Aw4JIUxK3/8aoB3JQ6KvgLt+6P3rx7HrDbsSN0Y+GvIRa+etZdnjy6i1fS3av9aemo2qfyV5zI0svi7ZAq/Vta1Y9eYqZh8xmwb7N6j2zShbtGjBIYccwquvvsq8efPYfffdgWSb8pUrV3LppZcWbik4depUatasyeOPP56xre+++47ly5fz9ddfs+222xZez87O5le/+lWp8nvuuScdOnTg0UcfZfHixfTt25du3bqx3377UbNmxZ9rTk4O+fn5dO/enezs7HLLFpynftBBB2XcIrFXr148/PDDvP3225x66qnlthNCoEePHqXyatSoQffu3XnwwQd55513+OUvix8Ju//++1d4T5Ik/TeJMd4XQmgI/AWYFEKYQPI3ciSZb/QkCa5eGGN8uET1PsDtIYSFJMHqT0jO+25NEkCuAdwSY5xRslvgBWByCOEJkrPDDyI5LmkB8PsS5U8mmbPcE0K4AJhGMkdoAexN8vd5F5JAPzHGe0IIBwK/BeaHEMal85qn7+fvJMcyQfIS7lrgovRuVAVns4+KMRY/1yaz0SQvITwZQniKZG62V/r+nwBOqkQb5YoxfhlCOBl4EngjPTd7l2TOs3f6vnYtpwmAccAHwP+EEH5J8hnuRHKE1TMkc6mifgucHUJ4E/gPyQr2XYGjSV5+GFmkbEWf4Z/S4zwbODqE8BrJee47kPy30g34A9/vOqCt0Lezv+WTv37CxpUbaXNnGzau2sisw2ax/uP1tLigBTUaZg7bhFSgzZ1tiDHywekfALD7/clq9PJ8/frXfD76c3Z/cHeyspPNDdYvWs8HZ31A27+3JbtF+XOHqhg9ejR5eXlcddVVtG3btlR+8+bVeyJEwQryojt7FWjYsPIr8r/66isefvhhOnXqxEUXXVQsr1atWlx//fWMHz+eRx55hPbt2xfL32effRg8eDCSfloMoEuSpE2RKWQVy7heVr1vSQKqPUke0BxDsi3hf0gC2PeVql1+u2X1Xep6jPHZEELBg4XDSR6WfEHywObPMcYvMtSZFUJYQbJioWSQvODM80znnxfUvyGEMBm4ADgQ6Euyav0z4A6+P9Ow3LFXRvoh17skK7sPJvlsV5Cca35Phj5K9nM6yffQHxhC8oBsHMmKkafZ9O/xNpJAeWeSBzs1gE9Jto+8KZY+A15bqRACu928G2vnreWzUZ8B0OndTtTavvxt6grOPF//8Xrav9Kej6/+mFmHz6L9S+1p0LlBmfV+eXESeP3PJf9h5aSVrJm5hgb7N+BX//wVWXU3dYfQsg0YMIBXXnmFMWPGMGLECOD77dtPO+20wnJffvkleXl5GbfxKxBCYM2aNcUC6DvssEPGsqlUitdee43hw4fz1FNP8bvf/Y4YIw0aNGDgwIGF2w+WZeXK5EjPyjyUKtgOvlmzzA/bdtxxx2LlNkc7ZdWRJGkrUJm/wzOWiTGOTAdlLyTZiargfJNPSV5yHRVjzHTU0v+RbOd9aLrOsSR/Ly8FniVZQfxiGWMZCfwDuBg4keRF1XuBK2JyxFLR8X0WQugInA8cTxJQzyKZD80BbiEJKBetMzCEMB44i2T1dW2SQP3E9NgKyq0MIfQjmTcMJNn9C+BBvt+CvkwxxndDCD2AP5O8XFwDmEWyxflqkvlJWXPRSosxvhBCKDiHvBfJnPBrki3Sr83QdixR/7sQwiEkW/YfTnKm+XskAf6VJN9B0TqPkN65i2SVch2SeeAjJPOgOUXaLvczjDFuBI4LIZySzv81yYvZy0leUr4CKPlyRmXm6voRNT25KTE3Mm/QPDau3Mi6BetYv3A97V9tzzbtyj4+aVPFjZEv//kl7x//Pu3GtiN3aS4ze6ZPWcsvv25VTZs2DYA+ffpUb8NlOOmkk7jllls45phjOOGEE+jVqxfdunVjl13KPX2ulJycnMLjroYNG1Yq/7vvvgNg3rzS6zx8MVj6aTKALkmSqiTGWGqpYoxxERWccxdj7Fni943Ajen0Q8ZTZt8xxjF8f5ZcybwZJA+MqtJXxqhXjHEJyfaFFdX/N/DvSvbVqipjy1B/GsnDrfLKZPzsYozrgT+mU0mb/D3GGF8BXqmonP47rJ23ljUz1xT+vvyp5TQ/t/zA7QdnfFB45nmDzg2ov399Zh8xm9l9ZtN5fudyz0H/5cW/ZNWbq1jxdPIMeHMFzwH69evHkCFDeOihh7j22mtZsWIFL730Evvss0+xleMNGzYkxsiKFSvKaa208s70a9iwIX/961/561//yoIFC5g4cSJ33nkno0aNYtWqVdx3X9nvFjVq1AiAzz77rMIxFKzG+OKLUu8LAbBkyZJi5TZHO+V9DpIkbSmZ5jsZypT7t3o6QH5eFfudT7Lt+s1VqVek/oMkQdbKlP0WuC6dKtv+o2R+6bdkuZco42zxGOMwki3Iy6s/leQlgkwyzV16ZipYkRjjXJIAdEXlypprLiF5+SCTrBJlc4CcKoytzM+wSJmHKR0oz1Suwrm6toxmA5qR+3Uu/7n4PwB0nNGR+vvUL7fO/2fvzuOcqs/+/79OZmEWhl0gsggIiDsoAgUX0Lr1p1i1WrUKaK3e6t3eYlu1aimiovamX5e7dal1t6LVVqSLS1FBwA1RtCKgbMMWloEZGGZPcn5/fM6ZLJNkMkMyW97Px+M8INvJOUlmJp/r+lzXx13zfPtT2znsqcPY98E+Vl+5un5/8fQ4vQdHvX4UX533FZ+d8Bl1e+rwdPIw8r2R5A1MXfU5NG1SbyqMHj2aJUuWcM899/Dqq6/y/PPPY9s2I0aMYObMmVx88cVJ7Wf37t2ASaQvWxb7x9WyLCoqKhpcr4nBIh1To18IRUREREREwlWsqmDFpBXkHpTL+B3j6fc//fj2+m/Z+mjixO3guwdz7L9D1ebZRdkc88YxjHhmRMLkOUDZ4jL2vLWn/nLxXcXYdnoKafLz8/nBD37Atm3bWLBgAS+88AJ+v5+pU6dG3G/cuHGUlpayalV6OmQOGTKEK6+8koULF9K5c2dee+21hPcfM2YMHo+HxYsXU11dnfC+o0aNAmDJkiUEgw3LTt59910sy+K4445rdD+2bbNw4cIGtwUCARYvXgzQ6H5ERERERDJJXWkdO17YUX9526PbsIOJxzeb52zG96SPEU+PwHull+GPD8f7Yy+rr1zN3g8Td47qcUYPhv7fUCq+qqB2Wy3HvHkMeYekNnkOTZvUG4u7vFSsMYqbnI82duxY5s+fT2lpKUuXLmXGjBns2LGDyy67LOY4JRZ3wu/06dMJBAJxtwULGtYEaGKwSMekBLqIiIiIiDTJxpkb69c8z+2dy9AHhtLvf/qx4fYN1O2ui/u4guEFZt3yMNlF2fQ6r1fC5ytbXFa/5vlJFSdx6P87lE33bWLDbRvSlkSfNm0atm3z7LPP8vzzz5OTk8Nll0UWGU2fPh3btvnJT35SX2kdrrKysr6FYTI2btzIhg0bGly/Z88eampqYq7rF65Xr15ccsklbN26lZtvvrnB7RUVFezbtw8wFSGnn346Gzdu5IEHIgvdPv74Y+bOnUuPHj04//zzEz7n97//fXr06MHcuXMbnOsDDzzAhg0bOP300+nfv3/C/YiIiIiIZIq60rr6Nc+P//x4RjwzAt+TPr659puESfSDrz2Yo14/qr7a3F0T/fA/H06XcfGXxAKz5vmm2ZvqL6+dvpZAdSA1JxRm3LhxALz11lvNery79NXmzQ1XeItXGe7Kyclh3LhxzJw5k4ceeohgMMi8efPqb8/KMs0Y3Fbt4cInI4uIgBLoIiIiIiLSRCOeGsGx7x1bv+a5uyb66M9HN1pJ3hzbn90eseb5gOkDOPT/HcqOuTvwl/pT/nwAJ510EoMHD+aVV15hxYoVnH322fTqFZnoP/XUU7n//vv56KOPGDZsGBdffDG33HILN9xwA+eccw59+vSJuX5ePF988QVDhw5l3LhxXHnlldx+++1cc801jBo1Cr/fzy233NLoPn7/+99z7LHH8oc//IEjjzyS6dOnc8stt3DRRRdx8MEHs2LFivr7PvbYY/Tt25ebb76ZM888k9tvv50rrriCiRMnkpWVxdNPP01hYeQajNETFgoLC3nqqafweDyccsopXHHFFdx+++2cddZZ3HzzzRx88ME89thjSb8GIiIiIiIdne9PPqo3mDXPi0YW0XdqX0Y8PYLtz2xn3yf74j4uu2s2vc6NHJNYHos+l/ZJWAVdXVxdv+b5uI3jOOatYyh7t4yVF65MeRL9uuuuIysri1mzZrFmzZoGtzdWmT527Fhs2+aJJ56IuP6dd97hpZdeanD/Dz/8MGb3LXeJqby8UJV9z549sW07ZnL+oIMO4kc/+hGffvopd999d8wK+PXr17Nx48aExy8iHYfWQBcRERERkSbJKswiqzByKUXLstLSAhBg+GPDsWvtiDXPB0wfgPdqL9lF6RvSTJkyhVmzZuHxeJg2bVrM+/zyl79kwoQJPPzwwyxZsoT58+fTtWtX+vXrx3/9139x6aWXNnhMvODW6NGj+dWvfsWiRYt46623KC0t5aCDDuKEE07gZz/7GWeccUaD/UTvq1u3bnzwwQc8+OCDvPzyyzzxxBNkZWUxYMAArr76ao444oj6+w4ePLg+QPSvf/2LRYsW0aVLF773ve9x2223cfzxxyd17JMnT2bp0qXMnj2bt99+m71799K3b1+uv/567rjjjphrAqrNoYiIyIFLZk1xEWl7BvxiAL1/2Dti/fG+U/vSbWK3tIypdr9h1vce+d5I8g7JI++QPI56/Si+vvRrKldWUnR84rXXm+Lwww/nkUce4brrrmPkyJFMnjyZYcOGUVJSwrJly+jWrRvvvPNO3MdfeeWVzJkzh3vvvZcVKyB0TXcAACAASURBVFZwxBFH8M033/Dmm29ywQUX8Oqrr0bc/7e//S3vvvtu/QTozp07s3LlSt544w169OjBNddcU3/fSZMm4fF4uPXWW/nyyy/rq91vv/12wExGXrt2Lb/5zW94/vnnOfHEE+nTpw/btm1j1apVfPrpp8ydO5dBgwal7PVy7d69m5///Of146SSkhIArrrqqvrrfvWrXzF8+PCUP7eIxGalq+WhiIiIiIi0H5ZlHQcsX758udarljbrs88+cxP7x9u2/VlrH4+IiIiICLT98ZS/3N9g8nGs61Ll448/Zs6cOSxevJiysjJ69erFMcccw9VXX80FF1xAcXExQ4YMYdq0aTz55JMRj129ejW//OUvef/997Ftm9GjRzNr1izWrVvHVVddxdNPP82UKVMAWLBgQf1yUlu3bsXv99O/f3/OOussbrrpJgYMGBCx7xdffJE5c+awZs0aqqursSwLvz/U1czv9/PHP/6RF198kZUrV1JdXU2fPn0YNmwYkydP5vLLL69PvC9atIhTTz2VmTNn8utf//qAXq/i4mIGDx6ccKLxe++9x8knn9zovjRmEkkNJdBFRERERKTNB3xEQMEgEREREWmbNJ6StkJjJpHU0BroIiIiIiIiIiIiIiIiIiIiKIEuIiIiIiIiIiIiIiIiIiICQHoWuBARERERERERERERERHJMA8++CB79+5t9H4TJ07klFNOaYEjEpGmUgJdREREREREREREREREJAUeeughNm3a1Oj9LMtSAl2kjVICXURERERERERERERERCQFNmzY0NqHICIHSGugi4iIiIiIiIiIiIiIiIiIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIiIiIiIiIiIiIgIoAS6iIiIiIhIsyxatAiPx8OsWbPS/lzTpk3D4/GwadOmA9rPM888g8fj4bnnnkvRkYmIiIiIiIiIdCxKoIuIiIiIiDSTZVlYltVunqeljldEREREREREpL1SAl1ERERERKQZxo4dy6pVq7jhhhvS/lz33Xcfq1atol+/fge0nwsuuIBVq1Zx/vnnp+jIREREREREmqa4uBiPx8NVV13V2ociIhKTEugiIiIiIiLNkJeXx/Dhw+nRo0fan6tPnz4MHz6crKysA9pPUVERw4cPp6ioKEVHJiIiIiIi0vbNnDkTj8fD+++/39qH0sCbb77JT3/6U0aNGkXPnj3Jz8/nsMMOY/r06ezcubO1D08kIymBLiIiIiIiLaqmpoapU6fy2WeftfahRPj444/xeDxceOGFce8zfPhw8vPzKSsri7sG+sSJE8nKyqK2tpYZM2YwfPhwOnXqFFFdsW/fPm688UYGDBhAfn4+hx9+OA888AAbNmyIWYkRaw308KqN4uJiLrnkEg466CDy8/MZPXo0f//73xsc/7PPPht3DfStW7fys5/9jOHDh1NQUEDPnj0ZO3Ysd999d8T9Fi5cyDXXXMORRx5J165dKSgo4KijjmLmzJlUV1cnfpFFRERERERaQVtdzqqmpobvfe97PPHEE/Tu3ZurrrqK66+/noKCAh566CGOPfZY1q5d29qHKZJxslv7AEREREREpP0KBoN4PE2bl/vUU0/x3HPPsX59MYsXL0zPgTXD2LFjGT58OP/6178oLS2le/fuEbd/+OGHrF27losuuohu3brF3Y8blLnwwgtZvnw5Z599Nueffz59+vQBTIBk0qRJfP755xx33HFcfvnl7N27l9mzZ7N48eKYQZ1EwZ6NGzcyZswYDj30UKZMmcKePXt4+eWXOf/88/n3v//NpEmTYh5fuE8//ZQzzzyTsrIyTjnlFC688EIqKir4+uuvufPOO7njjjvq73v//fezZs0axo8fzznnnENVVRVLly5l1qxZLFy4kHfffbfJnwkREREREZF0sm27tQ8hpqysLO6++26uv/76BuPM6667jscff5ybbrqJ+fPnt9IRimQmRTVERERERKRZdu3axaBBQ/jLX/6S9GNqamqYNWs2MJwlSxaxcOHCJj1neXl50w6yiaZOnUptbS1z585tcNuzzz6LZVlMnTq10f3Yts3mzZtZuXIlTz75JPfffz833XQTAL/97W/5/PPPueyyy/j000+59957eeSRR1ixYgVLlixp8jEvWrSIn/70p3zwwQf87ne/4+mnn2bevHkEg0HmzJnT6OPr6uq46KKLKCsrY+7cubz77rvce++9PPzwwyxYsICNGzdG3P/RRx9l/fr1vPDCC9x///08/PDDLF++nNtuu43Fixfz6quvNvkcREREREQkebfddnub/t69bNkyfvjDH9K/f3/y8vI4+OCDOfPMM3nllVcSPm7ixIlxJ+PG66a1YsUKfvjDHzJo0CDy8vLo1asXxx57LDfeeCOBQACAwYMH13cOc5/D4/E0WCKrqqqKe++9l1GjRtG5c2eKiooYP348L730UoPjCe9I9vHHH3PWWWfRo0cPsrKyIjqHNSY7O5vbbrst5iTtGTNmADR53CwiB04JdBERERERaZY5c+aweXMxN998G36/P6nHPPXUU+zYsQ14naysUfz61zOTfr7i4mL69vXyhz/8oXkHnIQrrrgCy7J49tlnI66vra3lL3/5C7179+ass85qdD+WZXHXXXc1qGIHE/jJyspi9uzZEdf369ePG2+8scmVEYMGDeL222+PuO6MM85g4MCBLFu2rNHH//3vf6e4uJjzzjuPiy++uMHt/fr1a/B8sUyfPh3btnn77beTP3gRERERkQwWDAab/JhPPvmEe++dzTXXXE9FRUUajurAPPHEE4wfP5758+czYcIEfvGLX3DOOeewc+dOHn300YSPbazNevRtX3zxBd/5znf45z//yfjx4/n5z3/OpZdeysEHH8xjjz1GTU0NYMYqEydOBMzyWDNnzqzfXHv37mXChAnccccd5OTk8OMf/5hp06ZRUlLCZZddVp/MjrZ06VJOPvlkAoEAV199NVdccQW5ublJvFKNc/eTk5OTkv2JSPLUwl1ERERERJps165dPPzw74HvU1w8jxdeeIFp06YlfEyo+vxSYASBwEyWLDmPhQsX1gczErnnntlUVlYwc+bdXHXVVeTn56fgTCL179+fU089lXfeeYfVq1czYsQIAF5//XXKysr4+c9/nnR78jFjxjS4rry8nPXr1zNw4EAGDhzY4PYTTzyxycc8cuTImEGmAQMG8NFHHzX6+I8++gjLspKaGABQWVnJgw8+yLx58/jmm28oLy+vT/pblsXWrVubdgIiIiIiIhkoGAwyYcLJnHrqKdxzzz1JP+43v7mTrKxBlJVt4dFHH+UXv/hF0o+tqKggLy+vQeV1qqxatYobbriBrl27snTpUg477LCI21M9Vnjuueeora1l3rx5nHvuuRG37d27l4KCAgB+9rOfUVpayqJFi5g2bRonn3xyg339z//8D1988QW/+93vuPHGG+uvr62t5bzzzmP27NlceOGFHHvssRGPW7BgAY8//jhXX311Ss8N4E9/+hMAZ599dsr3LSKJqQJdRERERESabM6cOdTVZQF/wrLOZ+bMuxutQnerz23bXU/73KSr0IuLi3nqqaeA69i9exePP/74gZ5CXFOnTsW27YgqdLd9+5QpU5Lej7vmebh9+/bFvS3R9Yl07do15vXZ2dlJVbSUlZUBDSvNY/H7/UyaNIk77riDmpoaLrnkEm677bb66g3btuurPEREREREJL6//vWvfPTRUn772zls2bIlqcd88sknvPnmvwgEZmPbVzJ79m+TrkKvra3l6KNH8eMfpz7R63rkkUcIBALMmDGjQfIckhtzNIU7kTjW5Op446RY9uzZw5///GdOOOGEiOQ5mCrw+++/n2AwyIsvvtjgsSNHjkxL8nzZsmXMmjWLoqIi7rrrrpTvX0QSUwW6iIiIiIg0iVt9HghMB3pi2zMoLh6VsArd7/cza9ZsbLsn8FD99YFAJ5YsWcT7778fswrAdc89s7Gs7sD/YtvV3HPP/Vx77bVpqUK/4IILuP7663nhhReYPXs2JSUlvP3224wcOZKjjz76gPbdpUsXAHbs2BHz9njXp5O71l4y1SCvv/46y5Yt48orr+TJJ5+MuG379u0RLRBFRERERCS2YDDIr399Jx7Pydj2V9x33338/ve/b/Rxv/nNnWRnj8Dvvxj4DmVlTyddhf7ss8+yYcO3bNy4jltvvaW+21YqffzxxwBJd7c6UJdccgkPPfQQ5513HhdddBGnnXYaEyZMYMiQIU3az7Jly+rXS7/zzjsb3F5bWwvA6tWrG9wWq/PYgVqzZg3nnnsufr+fuXPnMnjw4JQ/h4gkpgS6iIiIiIg0Saj6fLpzzcj6KvTLL7+c7OyGwwzbtjn66KM46KCdwCcRt2Vnn5BwjTi3+jwQuBcoBG5n9+7nePzxxxtUB6RCfn4+P/jBD3jmmWdYsGABX331FX6/n6lTpx7wvouKihgyZAjFxcVs2rSpQRv3xYsXH/BzNNW4ceOwbZu33nqLa665JuF9165di2VZXHDBBQ1uW7hwYZqOUERERESkY/nrX//KmjUrgaXAQh5//E5uvfVW+vfvH/cxbvU5vAhkAYPqq9Cvu+46CgsL4z62traWO++8B8v6PllZn3LXXXfz5z+/kOKzalp3q1QYPXo0S5Ys4Z577uHVV1/l+eefx7ZtRowYwcyZM7n44ouT2s/u3bsBk0hftmxZzPtYlhWz2r9v377NP4EYVq9ezaRJkygrK+Pll19m8uTJKd2/iCRHLdxFRERERCRpNTU1PPzw7wkGA+TknEhOzuHk5ByOx/MhxcXrmD9/fszH5eTk8Pbbb/Dll8sbbJ999gnjxo2L+5yh6vPrnGsOxbancM8991NVVZX6kwSmTZtW38b9+eefJycnh8suuywl+54yZQqBQIBf/epXEddv3ryZhx56KOZ65ul07rnnMmjQIObNm8crr7zS4PbwyvRBgwZh2zbvvfdexH3Wr1/Prbfe2uLHLiIiIiLS3oSqz88AxgP/jW135r777kv4uFmz7gYs4D3M2Og6YDulpY0vcfXss8+ydesmbPsu/P5fMXfu3JjV1AeqKd2tYvF4TMoq1lJUbnI+2tixY5k/fz6lpaUsXbqUGTNmsGPHDi677LKkJ/m67d6nT59OIBCIuy1YsKDBY1M5Blq5ciUTJ06krKyMV199le9///sp27eINI0q0EVEREREJGk5OTnMnDmDnTt3NrgtKyuL448/PqXPV11dzdNPP43f78fj6VV/vW37KSnxM2/ePC699NKUPifASSedxODBg3nllVeoq6tj8uTJ9OrVq/EHJuHmm29m3rx5vPTSS6xevZozzjiDsrIyXnnlFU455RRee+21+sBROti2HXE5JyeHV155hTPPPJMf/vCHPPbYY4wZM4aqqiq+/vprFi5cWN+y8Nxzz2Xo0KE88MAD/Oc//2HUqFEUFxfzz3/+k3POOYeXXnopbcctIiIiItIRhKrP/+hc04VA4OeNVqEPHjyYo48eBSyPuuW4hGOVUPX5Rdj2UcAwsrJmp6UKfdy4cSxfvpy33nqL4cOHN/nx3bt3B8zk4kMOOSTitniV4a6cnBzGjRvHuHHjGDp0KFOmTGHevHlMnDgRMONVoL5Ve7gxY8bg8XhapSOY68svv+S73/0u+/fv57XXXmuxNvgiEpsS6CIiIiIikjSPx8Mtt9zSYs/XqVMnXnjh+bgJ+zPOOCNtzz1lyhRmzZqFx+OJu7a7ZVkxKw4SVSHk5eWxcOFCZsyYwauvvsqDDz7I4MGDueOOO5gwYQKvvfZafQVEY/uM9/yNPSba8ccfz4oVK7jvvvt44403WLp0KUVFRQwdOpS77rqr/n4FBQW899573HrrrSxcuJAlS5YwZMgQfvOb33DjjTfy8ssvqwpdRERERCSBOXMeACxycn5cf51tV+P31/LII48we/bsmI/7v/97qFnP51afwz+cazrh99/G3Lk/5de/viOla6Ffd911PPbYY8yaNYszzjiDww47LOL2rVu3JmzvPnbsWP72t7/xxBNPcPfdd9df/84778ScrPvhhx8yatQo8vLyIq7fvn07QMT1PXv2xLZtNm/e3GA/Bx10ED/60Y944YUXuPvuu7ntttsaTGpev349Ho+HQYMGxX8BmmnFihV897vfpbq6mvnz5/Pd73435c8hIk1jRVcfiIiIiIhI5rEs6zhg+fLlyznuuONa+3Ay1hNPPMG1117L448/zk9+8pPWPpw257PPPnO7HBxv2/ZnrX08IiIiIiLQtPHUCy+8wOeffx7ztosuuijh8lbNMXLkcXzxxed4PKFksm0Hse1abrrpJn73u9+l9Pn+9Kc/cd1115Gdnc3kyZMZNmwYJSUlLFu2jG7duvHOO+9QXFzM4MGDmTZtGk899VT9Y0tKSjjyyCMpKSnh7LPP5ogjjuCbb77hzTffZPLkybz66qs888wzTJkyBYDzzz+fd999t76DWOfOnVm5ciVvvPEGXbt25ZNPPmHIkCGAWVv8qKOOonfv3lx22WX11e633347AOXl5Zx11ll89NFHDB06lBNPPJE+ffqwbds2Vq1axaeffsrcuXPr11VftGgRkyZNYubMmcyYMaPZr1dZWRmHHnooZWVlnHbaaUyYMCHm/aZPn06XLl0a3Z/GTCKpoQp0ERERERGRFubz+fB6vRHXbdq0ibvuuovc3FwmT57cSkcmIiIiIiLpdPnll3P55Ze32PM98MDv+Oqrr2LeduaZZ6b8+a6++mqOPvpo5syZw6JFi3j99dfp1asXxxxzDFdffXX9/WJ10+rVqxeLFi3il7/8Je+//z7vv/8+o0ePZsGCBaxbt46//vWvEfe/4YYb6NGjBx9//DFLly7F7/fTv39//vu//5ubbrqJAQMG1N93xIgRPPfcc8yZM4dHH32U6upqLMuqT6AXFRWxaNEi/vjHP/Liiy/yt7/9jerqavr06cOwYcN48MEHOf300yOev7GOYMnYu3dv/fru77zzDu+8807M+1155ZVJJdBFJDVUgS4iIiIiIqpAb2Hjx4+nrq6O448/nm7durFx40b+8Y9/UFVVxf/+7/9y0003tfYhtkmqphARERGRtkjjKWkrNGYSSQ1VoIuIiIiIiLSwqVOn8uc//5l58+ZRWlpK586dGT9+PD/96U8599xzW/vwREREREREREQylhLoIiIiIiIiLezaa6/l2muvbe3DEBERERERERGRKEqgi4iIiIiIiIiIiIiIiKTAgw8+yN69exu938SJEznllFNa4IhEpKmUQBcRERERERERERERERFJgYceeohNmzY1ej/LspRAF2mjlEAXERERERERERERERERSYENGza09iGIyAHytPYBiIiIiIiIiIiIiIiIiIiItAVKoIuIiIiIiIiIiIiIiIiIiKAEuoiIiIiIiIiIiIiIiIiICKA10EVEREREJMyqVata+xBE4tLnU0RERERERETSTQl0EREREREBKPF4PNWXX355XmsfiEgiHo+nOhgMlrT2cYiIiIiIiIhIx6QEuoiIiIiIYNv2JsuyDgN6tfaxiCQSDAZLbNve1NrHISIiIiIiIiIdkxLoIiIiIiICmCQ6oMSkiIiIiIhIM2jJIWlt+gyKpIZl23ZrH4OIiIiIiIiIiIiIiEi7ZFnWQI/HsyYYDGpJLGl1zrJXh6lzl0jzKYEuIiIiIiIiIiIiIiJyACzLGoiWxJK2QcteiRwgJdBFREREREREREREREREREQAT2sfgIiIiIiIiIiIiIiIiIiISFugBLqIiIiIiIiIiIiIiIiIiAhKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiABKoIuIiIiIiIiIiIiIiIiIiACQ3doHICIiIiIdW3l5eRZQAOQ7/0b/3926BgKB/tXV1dW2ba8Etjjbdq/XG2iVgxcREREREUmz8vJyC+hE7PFS+P+72Lbdpaampp/f7/8aKMYZN3m93v2tcvAiIiIdkGXbdmsfg4iIiIi0E+Xl5TkkToTnA4VAN6CLs3UGcjCTN3PC/u9+EbWcf+v8fn/36urqHKDUuW4/sAf4BthAKKm+z+v16ousiIiIiIi0KeXl5R4gj8YnERdhxk1dMWOmToTGS+5mETluCti2HaiqqjokGAyWAtVALbAP2AysIXIisj/d5ysiItIRKYEuIiIikoGcCoccEifC3aBOF0xQp4tzXX0ivKqqakJ2dva2nJycbc6u3QBPnbP5w/5fF3V9A3V1dYfU1NT0ApZjlhvqTCgR7wGGAduAj4HVmCDRFmCb1+utS8FLIyIiIiIiAtR308oniW5azlbkbLk4Y6a6urrBfr+/X35+/vKwXVtAgNhjpfBxVDD6mGzb9lRVVY0NBoPrgR2YxLs7ZsoHegK9gAU0nIi8VxORRUREGqcW7iIiIiLtXFS7v3iJ8AJMMtqtcOhC7AoHD5EVDkEaJsKrgXKgLhAIeC3L+jYnJ2dlCk/JrUgPYiop9oXddppzDEOAo5zrKoEyn8+3FlhHKDi0R8EhEREREREBKC8vzyb5blpFhCrDo8dMWVG7tog9cbgK8AcCgYGBQKA3kMoxU7gaYJezAYwBjgA+AiZixlB+zLjK5/P5oici16bpuERERNotJdBFRERE2hAnGR5d4RCr9V8XmtHuj4aBnUoiqxzaw1rjVcB65/8WJsjVBRgPnEwo8b7DCQ5twgSHtnq93uqWP1wREREREUklZ2mpxrppud2s3OrwAiLHS9nEWVoqaqul4bipKRN1G1SRp0Ci53dv2+psYCrii4BBwJHOde5E5HVETkTerYnIIiKS6ZRAFxEREUkTZ+27xhLhhSRo9xe2QcNkeHRgZz+NtPtLk1QHVxrbnx31//3O5raRz8YEyvoBh2Ferypgn8/nWw98Syg4tMvr9bbU6yQiIiIiImGcCcS5JNdNq2vYlkdkItytDA8fM4UvLeVuNYTGTXGXlkqDVI+ZrMbvEqEW2O1s7uMLMOOmccBJhCYi74wxEbkqFQctIiLSXiiBLiIiIpKERtr9uf8vtG27RyAQGOzxeGyPxxMrEd6kdn9hl9tqBUBTAzct8Xx+YI+zudzg0HGYAJGNaUNf4gSHinGS6l6vtyKlRywiIiIikgGcZHgeSXTTCgQCA4FuWVlZAULJcDcRnmhpqfDEdxUm4ete1x66aaVSvDGileC28MdWOJvPuS4bM6HbCwzDvAfuROQNRE5E3qmJyCIi0pEpgS4iIiIZJ0G7v+gKhya3+7Nt21NdXX1+bm7ugtzc3HUceLu/TJOuhHyls213LmdhgkO9gLMJVfX39Pl8y4BPCQWHtnu93kwLxomIiIhIBgvrptXYJGJ3vNTF2XKJTITHXFqqtrZ2dDAYLCwsLJyPGSNVEJkgV3I2vnSNmfxAqbO53OXDRgJjMe9jPlDh8/n+DWwkNBF5f5qOS0REpMUpgS4iIiLtVoJ2f9EVD+Ht/oqITIanut1fofNvCaFkrTRNogkGyVRTJCMAlDmbqzdwsfMcw5zr9gN7fD7fN8AGQkn1fVoXUERERETag/Ly8iwSj5fCl5ZyJxF3pmE3rVhLS4V3zXL/Xx11XYNkuG3bh2HGYtuib+uAWqNrV6rGKlXOtsO57AF+iPl8WM7lWkyV+mZgDZETkVuqRb6IiEhKKYEuIiIibUKcdn+xAjtuQMetcFC7v9bX2uv5pUqN8+82YB3mc+R2IjjVuezHfG62Oa3fN2OCQz6v11vb4kcsIiIiIhnFWVoq0XipgFAy3B03FdIwER7eTcvlJsDDk+JVqJtWKnSU1y2I+RxUA18713XCfM5GAKMItYcv9fl83wLrCSXV92oisoiItAdKoIuIiEjKOe3+8ki87l10u78izMC70XZ/NKxwULu/1tNaye50cM/F/bwFMcnyfWH3ycV8VgcDRznXVQJlPp9vLSbx7gaH9ig4JCIiIiKxOBOI3aWlEq0ZXgh0IzRuyqdhMjxeN63wRHgtkeMmVQbLgQgf59QAu5wNzGfQnYh8CmYysjsR2Rc1EXmbJiKLiEhbpAS6iIiIJNRIu7/oCocG7f78fv9BdXV1R+Tl5X1lWVZ4xUJ4u7/wwE6j7f6kw2uJFu7NVQvsdjYwx1OA+ex/BziZUOJ9hxMc2oQJDm31er3VLX7EIiIiIpJWTjK8E4kT4fmYiZjdaNhNK6e6uvp4j8dTnpubuzVs1243rfDxkjtmKic0ZlIyXMKle8zU2CRqG/P5LAfcz3MO5jM/CDjSua4S2BtjIvJuTUQWEZHWpgS6iIhIBknQ7i+6wqG57f6iAztVwWCwWyAQODQYDM7Lysran9YTlI6itaraoyvQk+G2J6wAfM512Zjg6MHAcGe/1Zh1Abdi1gXciAkO7fR6vZokIiIiItJGhC0t1Vg3reilpTrRcMwUvrQUhCrDw7dKoC4QCBxs2/Y3wMr0sipKTAAAIABJREFUnqG0gnSNb1ozydzU564j/kTkccBJmG5z5cBun8+3DviG0ETkqlQctIiISLKUQBcREWmHErT7i654CG/3V+Rcl0y7v+hEeHi7PzdRnuyAWRW3HV8qAzeNVUu0dgV6MvxAqbO58jHBoV9h2hX+B9gPlPh8vjVAMU7Fhdfr1UQTERERkRRwlpZqrJuWu7SUmxAvwizbE54Ib2xpKXfbT+RYqikTJYO0/e+50nwdacyUigkBiSYinwz8EvgboYnIG4BvCVWpayKyiIiklRLoIiIirSxOu79Yrf/cdn9FmMBOfbu/sC26wgEaBnXcdn/h7dNFmqsjr4GeSlXO5sesDbgGs9RBT+BMzM9uLSY4tNm53Q0Obfd6vfo5FRERkYzmLC2VTDctNxEesbQUoUR4DpHf98KXlooeN4VfVnJbmsW2O+RHJx0nFT4ROQh8RWgi8khgLKH28Huc5bI2EqpSL0/DMYmISIZSAl1ERCSFYrT7i7cGntvmz610aKzdn7v2Xax2f+HBnkCaT/FAdKREq8iBCgB7nc3VCfP7YAQwilBVRqnP5/sG2EAoqb5X6wKKiIhIe1VeXp5MN60CIrtpucnw8ER4+NJS7ngjesxU3yad0ATitvo9SmMmaQ9a8nPqTkTe4Vz2YH4fdAfOQBORRUQkTZRAFxERiSOq3V+8RHj42neNtvurrq4eGQwGuxUUFLxLw6BORdTljtKOrK0Gp9Il0863paWzHWE6K9DDnyPe/msw1em7nMseQlVUE4HTMAHffYDPqbjYjAkObfN6vbXpO2wRERGRhpwJxLkk100rfL3wAiIT4fVLSwUCgcKqqqoz8vLy3s/Ozt5OZOK7BtMmPfy6jkLjiI4r5QnnRqraW2LZq9YaMwVp2kTktcB6Qkn1Mk1EFhGRZCiBLiIiGSGs3V+iRLi79p0b2Cmiee3+/MRp92fb9mDbtguAVWk7WZGWpwBE8poSzApi2hOWA1ud63Iwv58GAUc611UCZT6fbx2wjlBwaLeCQyIiIpKssKWlDrSbVvTSUtHdtNzxk7u0VEQ3Ldu2ewBn27ZdDBSn8ZRFWlJH+l6e7gr0pk4AiJ6IbGE6VnTBrKc+idBE5O1RE5G3aiKyiIjEogS6iIi0O067v0SJ8HxM1eaBtvtzAzupbPdnk7lt+TL1vDu6ln5f23sF+oGqA3Y7G5hjdgPZ4zBV6scCnwJrnODQJkLBoaoWP2IRERFpcU43LXdpqURjpyJC46bOxE6Gh3//sjCJ7vDxUvSY6YC6aVmWFT1GyySZeM7SfPHGLe29Av1Aueukx5qIfAhwODDUud9Sn8+3HlhLaCJyiSYii4iIEugiItJqnAoHd+27eIlwN6iTVLu/qKeIToS3lXZ/mRYU0cBTkmbbdksEe1pTqs/PbU9YAfgwvyvPBT4DvMAwTAVYFWZdwA3At4SCQzu9Xm9HWS5CRESkQ3K6aeXTtG5anYlcWipRN63ojlrRY6bW+q6gcZNIbK39s5HuMVs69h89EfkITOFFLjAGmECoA9hOn8+3BtMBYwuwRRORRUQyjxLoIiKSEo20+4uocLBtu5vf7x+RlZXl93g8Nsm3+wsP7MRs99cOZHJQpLUH+dJG2bZt2badHwwGC4LBYG9MICOLlv+5bu010FO1fzC/H8NbnuZjgukjgbGEqjL2OFXqGwkFh/an8fhEREQyWnl5eTaJx0v13bQCgcBA27Z7ZmdnV9FwzJQVY/duZXj4VkXkWKo9jEfawzGmi8ZMHVNK3lfbtnOcMVMXZ6mDglTst7mHk8Z9t8SkancCkc/ZwORKioC+mAp1C1OIsc/n820EviE0EXmHJiKLiHRsSqCLiEgDCdr9RVc4NLfdX7CmpuaS3Nzchbm5uato2CK9vSTDmyMTW7hncgAsU9S/x7ZtY9t2rm3bBcFgsMC27ULbtgvCNyD633wa/lx8B9iGWZvO3SpomWBKexZvEkCVs+1wLnswv8O7A2c4l2sxwaHNwBpCwaHtXq+3tTp2iIiItFnO0lKJKsLzMeMktyq8q3N9+Hgpm8ilpcD8Pa+rra0dGQwG+2RnZ/8F83e6glAi/ECXlmrLbKjvTJRJOur7KUbE+2vbtscZL8XbCqPHTjSM5Z8A7MWMlTY5/+6k5SbttmexXiM/UOpsLnci8jGY19vGdOnY7VSpbyQ0Ebk8zccsIiItSAl0EZEOLk67v1iBnfD1wotIrt1frAoHt92fG9iJNSPXA2DbdhlmcJdpOsJgUzKEbdtZ4YEd9/+YQE4h4PH7/WMrKiqODwvsxKqIqgUqLctyt1LLsraGXa60LKsyEAj08Pv93TATawYARwLjnX2UYn439QN6A7tIbWCoI1Sguxp7jiAm2LY37LpOmODQCGAUofbwpT6f71tgPaGk+l6tCygiIh2F000rl8TjJXcCsZsI7+JcH54Ij15ayv27Hz1mqia5paUGAz2B7ak613ZG4yZpF5yOWp3iTSAOBAKHAoUVFRU/dsZMhZjv3tGCOGMmQmOnPUSOoyosy6qtqakZjvk90g0zbjoCE2tx4zI5mCrqLZjfOSk/7TTs09VWxkyQeCLy6YRe830+n28LsJrQmMmnicgiIu2XEugiIu2I0+6vsUR4IWYA5a4X3pnG2/25ratitfsLvz5VA5jwgFKmyeSEUya+322KE9jJCwaDDYI60dXhTnK8ABNMjhYEKpzAjmVZVq3H49kSFuip8Hg89UEe5/9JdZawbTsb81n5HPjEuboLJig0ADPr/zBMkrcGE5hwK9S3YBL1bV1brQapwUxK2BW2r86Y1/8U4FTM34R9QLnP5/uaUKX6Nq/X2x5eexER6eCcZHisbloNlpYitGZ4kfOY8ER4oqWlosdM+8Iup7KbViZ2rwKNmTJJmzxft1V6+ATi8DFSrM5axD6XaszE4GwAy7J2hY+TosZLlZZlVVtW4y+Jbds5tbW1/W3bXkdoTe9swAsMBI51Lv/IuW0XoQr1zcCe5r0y9VrifWuJFu7NkWgi8nDMaw9mIvJ+n8/nA5YTSqqXaSKyiEj7oAS6iEgraaTdn/v/prb7cwcA0UGdWhq2SW/NL+yZnkDPqPO2LCtTB4dpfZ9jtEpvUO1Acq3SAaqc5HeFE7jZkSCwU2FZVq0b2Nm/f/+vs7KyPs/Ly1uWolOLdXz7gJXOdizwASYA5CbVxwITMT9fO4lsYVjWjOduqwnupuw/FefgrpNeDmx1rsvB/E36Mea1/hLz92Wvz+dbC6wjFBzareCQiIgcCGdpqehuWrHGTl2JTIbnEpkIz4nadaxuWn5MwiP8utZe3zbjxg6OTB0v6ntTGiTRKt0dKxWGJcNjxczriKwO32dZ1vawqvDocVOVZVlBgKqqqtMCgcBRBQUF81N0TrF+NvyEEuQeYBzwJKEx0wDgeOe+lUQulbWN+J0w4h5Gkw88eS3xs5/KKvd4E5FHA9cDf8X8zdkHbPf5fKsITQDf5vV6a1J0HCIikkJKoIuIpFh5eXkBpk1WrAqH+nZ/dXV1g23b7pybm7uNprX7qyG5dn9tVtiM6kwLiLgy9bwlgUZapcdbAy+ZVulllmVtiw7qRAV2DihwkIZJEsm0Hy92NlcvQoGhQZhgBZjfl+HBIR+prQxrqpZajzBdz1GHqXLxAyWYSQ0FmL9v44CTMK9vObDLCQ5twgSHtnq93qo0HZeIiLQz5eXlQzAtcKO7abmJ8C7BYLBHXV3dYTk5Ods8Ho+fyGR4+LgpvGuW+//qqOtaOxneHHYGrgMeLpPPPRM0+ftqVKv0hJOIw8ZMeXGeu9KyrApCY6dSIsdR9ROIPR5PlWVZdQd0tq0zQWKPs33hXM4D+hMaN52C+Z0axCTRw8dN+xPstyUmBbdEBXq6/i64E5HdKvWvCE1EHggc7lxfhZmIvA5YS2gicokmIouItD4l0EVEUq83ZoZpLxK0+/P7/YcFg8Feubm5/yE97f7aukyvpshEGfV+27ad6/f7e8ZaAy9GYKeQxlulu4GckqjLlR6Pp8L5t8qyrJaeVJOO9zXRPuMFU0qc7XPncj6h4NBATPvxbMzv2W2EKtS3YKrNwp833T+nbWU9v1Q8h7tOegVmcgKY17kI6IuZUGZhkhj7fD7fBuBbQsGhnV6vtz0mNERE5MCdhfn7HP03y02A+23bzq+rqzvF4/G84vF4ttA2umm1JI2ZMk9Gvd+2bVuBQKBrjFbp7ngp1iTiuK3Sw5LfJe4Y6UBapbdj8cZM1ZhE7VrnsgcTw3IT6kcA33FuKyOUTN+E6T4Vvk+NmZrGnYjstty3CE1EHgNMwIz/y4GdPp9vDWbCuDsRubIFj1VERFACXUQkHYKYZNh6TFuseCoJfTnORBkbDMrAKpJ2HQCL0yo9blW4sy4edXV159fVNShSqIpKfu9sJLBT08EDO+lUhUnUfutc9mASum5w6BhMkAJMVcZmItexS5eWauGebomqQvxAqbO58jHBoZGYlvtuVcYen8+3GthIKDiUqX8XRUQyUR2wJsHt3Z1/q0g8tuqoMnLMFNbdKNPOvb2Pmzy2becHg8GYE4hjLDPVBbCqqqpujNqVn8hJxOGt0sO3iuhW6dJkQWC7s7lLchUR2fb9SMxYqhbzfX0zJvGbzrbjLdXCvSWeI97PdayJyFmY178PoYnINZiJyBuBb4iciJxJRTgiIi1OCXQRkdRzK/Ia+zKekcGQMJmYSHZl6nm3CUm0So+1Bl7CVunOv2WWZW0D6gKBwElZWVlLs7OzvwlLiFcrsJO0xoKHzQkuum0JtwEfO9d1IVSh7ibVAaYSqlDfjFn/u7YZzxlLe2/hHv48TXmOKmfb4Vz2YIJD3YEzCAXl9vl8vs2YZIobHNru9Xrb3XIlIiLSqGTHTPHW+80EmT5mzMRzbxPn7Ewizos3gbgZrdLDJxGXApXBYPBQ27bzcnNz33AnEDv/Hmir9LaoNd7X5o4HyoGvnQ1M/uBgQuOmMZjJsX2B64gcN5VG76yZWqqFe1t7jgCm8r8s7Lo8zLj1aOAEZ3/7gd1OlfpGnHGTJiKLiKSWEugiIqkXJLkvyEHayOC4lWRqMKhdVxUcoJS/384aeNGBncbWwOsUY1dBIgM7FZZl7Sb2GnjuvzETesFgsKCysvKkrKysLTk5OZtSfc5tVKo/1421cE+VfZg1vFc6lwcCVwIrMInd7wCTMOe3g8gWhi1Rrd4cbTWBHi2IeQ3DX8dOmODQCGCUs/8hwNc+n+8jTGcXN6lepnUBRUTavWQmR7mTDzNx3AAaM0mK2Lad7VaGx2mV7nbScq/Lx0xwjBarVXpEVXjU2KkmrKNAhMrKyi5Az9zc3HVpO/G2JR2f63j7TOXvDT9m/LMJWOpc92Pn353AIcBo53IFkeuo+0jud31raKkE+oGqdradzmUP0BnoBnwXM9m/B5Dl8/kWYCYiu0uV+TQRWUSk+ZRAFxFJvaZUoMcakGaKTA4GZdp5u5VDie+UuFV6vDXw8on9elYRGdjZGRbYiQ7qqFX6gUn1C9eab4QbXPgc08bQAnoRal84BDPrH0xlRnhwaDumYqAxLRGooQWeIx3nUQPscjYwwaAfYF7rXMxkBj9m4sN2p/W7Gxza6vV6U9UlQEREWkbSFehk6LjJSTxm4pdUtXBPdKdQq/S4k4hjrBueE2NXfkKTiCsStEoPHzupo1bb0No/GwHMd/K/O5fziGz7PhHzmQtgOoCFj5sqkti/xkzxBTGv/b6w6/4/zFi1GDjWec5KoNTn832LJiKLiDSLEugiIqmX7GA/k1uYQ2Ymkl0Zcd5Oq/T8YDDYHcDv9w/z+/1BGrZKDw/sxPpuUodTFU5Yq/Q4QZ0KtUrvEBIN6NMZTImu3rYJJXQ/c64rAPoTamF4GuZz6ycUHNqECU7EWqu1pVq4p1tLBbXArFG/yvl/DqZK/RDgcOc4KoG9Pp9vPbCWUHCoRMEhEZE2LZmOXJmaSHVl8pgJMuDc3VbpztgoJxgMdqupqRkZa2mpZrRKLyPUXStWR62O2CpdjJb4rh6+/2rgW2cDM+mpD5HrqI93bislMqG+M8axtpfq8GSeoyXGI0FM3GR92PMWYsZNJ2EmIruTHrb7fL5VhCYib/N6velcz15EpN1SAl1EJPXUwj05mRoMapfJnDit0iOqHWikVbrf7z+Lhq3SKy3L2oOpeGhSq3SRVlIJfONsYKqk+xIKDh0LTHBu201kcKikBY6vvbRwT/Y5iHqeOszrujvsPgWY4NAYzGsfxFSt73TWBSwmtC5gVZqPWUREktdo55awSZGZOG4AjZnaHadVeoFt24UJWqVHTyL2hD3+oLq6ukMxnXnCk9/RrdLru2s11ipd2oSO9nPc2PkEMe3bfcAnznVdiKxSPwrz2a/BfFd3x0xb0nC8sbTFNdBT9TzuOun7MZO8wUxELsK89iOcx1RhJiKvQxORRUQaUAJdRCT11MI9OZkcDGrV83ZaqecEg8EGQZ0YgR03Od6cVumVlmVVBoPBIr/ff06nTp3+mJ2d7cuAVumZONBs6XNuqQr0ZASArc72kXNdV0IV6m5S3cJUZmRh2uvtcB6T6sqfjp5Aj2Zj2kBWYAJ0YF7jLpiJDUOd/dQA+3w+30bM5Ac3OLTD6/WqY4WISOtoSgV6po6bWn3s0EraROeBOK3SG7RLp2mt0isty9pvWdbO8Mpwy7Iqa2pqzrIsa0dBQcF8y7KSWRpI2peONk5s6vnsA1Y6G5iflYMJjZvGYVq/u+3Hc4CjMUn1sgM/3AY6cgI9ljpMZ689Yde5E5FPIHIi8q4YE5FjdVcTEenQlEAXEUm9ZKskMjUY4srk80/pedu27YkO6gSDwUKa3yrdDezstSzLl2D9u6pkWqXX1dUN9Pv9WJZVlwHJc0m/9vAh2utsXzmXc4F+hNYDHAIMx/we3E5klfreA3zulkygp1tzzyWAaQ1ZGnZdPiY4dAwmQORWZex2gkMbCQWHyg/gmEVEJHkB1MK9MZk8ZiKVS56Ft0qP01XLHS8Vho2ZmtoqPXppqUqPx1MJJDUOqq2t9VuWFVDyXFKgPSwbVYdJ0BaH7bMnZsw0FjgIuMC5bT+RYyYfSXQxaUSmJdBjqXS27c7lLEyVem/gUCInIhfTcCKyfleJSIemBLqISOolW4GuFu4ZeP5OW724591Iq/R4a+B1irGreK3S61v9tWCr9I420146rnQln2uBDc52CvA2JlDkti8cimk/DqYyIzw4tJ3QxKym6AjBoFS+H1XOtsO57MEEh7oDpwOHYapcFvt8vi3AakLBIZ/X69VSEiIiqZfsmCmlidR2JiPHTCQxcSKsVbqbEC+MnjicqFV6GLdVurvFa5Xujpmq09gqXeOmDixNk8lb8zOT6ue2MUtelQA9MHGOPwL9CXX2OhWTzwhgWpNvItT2vaKZz5lObT2BHi2AqfYPr/jPw0xEPgo4GRgJfAgU+3y+1URORN6XgmMQEWkzlEAXEUk9N4GekDPoztRWhJABwaCwVun1VeHBYLA3kFtVVXUq8QM7sV6XaiKT3w1apYcFdSqcNfBa7Fyl6ZYtW9a1e/fudUOHDm23rdCcz3iqNTb4b6kgRLq4x7/T2ZY71xdigkNuUv10TAWAH9PqfTMmQLQFkwhOtP+W0p4S6NGChDoFgEmgF2Je7+GYtvtu+8hSn8/3LbCeUFK9TOsCiogcsEbHTWGJykwdN3X4MRM0bJUeDAaLAAKBwIiqqqp+4VXhjbRKDwAVMVqlVwL1VeFhY6cqVXu3Xdu2bev0n//8p2tFRUV2r169qo877ri9nTt3bq/vV6p/jhvbX3uoQG9s/zZm3POts4H5W9CX0JjpGEzrcTCtycMnIu+i8XFlS2hPCfRYqp1tJ9AHONu53AU4DTNmrcNUqW8F1hCa1ODzer2pXrJMRKTFKIEuIpJ6auGenHZ3/rFapcdaB4/GW6UTCASOCQvs7LMsa3t4ZXhzWqW3A+3q/U6nnTt35r7++vJBvXrlVP73fw9am52d3d4Tcak+/tb6rLRU+/NY+6/ABBvWOJezAC+h4NAo4ETnthIig0MlYfvJtDXQU8WD+fu9y9nc5y/EBIdOAiZhAvP7gO0+n28VoeDQNq/XW9MCxyki0pEk8/1WLdzb2bk7rdI7uROIE7RKr++uRexW6QSDwaGWZZXjJMVjtUp3x0wej6eCJFult3Ht/gRSYfny5V0/+dvfBvUoKSno5vGw3rJY88kne86bMmVD7969a1v7+JqpvY/5orXGmCmIqTzfBnzsXNeFUIW6m1R3W4+Hj5m2YjqDudTCvencyWz7MW30XbmY92Eo5vUHM74tizERuVQTkUWkvVACXUQk9dTCPTmtGgxyWqV3ihfUiQrsFBK/Vbq7Bl4FsVulR1SG19bWHhcIBEZ37tz5wRY83damwVGUDz74qMfOnV0L9+6tyPv888+7nHDCCQe67nVryeTfYc2V7GsWIBRk+NC5rhsmKOQGiEY6+6ty7reJUFCoIwSDWjKBbtEwkeOuk74fE6QDU+1WhHn9RxB6/ff6fL51wFpC71uJgkMiIgkl8zsy2cnJHVWrJ9BjtEqPGCMdQKv0PR6PZ0v0BGJnq6mqqvp5Tk7Ovzp16vRlS55vK9P3BqC8vDzrk3/8Y8DI/fs7nTRixG6PZVFRW+v5+7ff9nz37bcrL7n88s2tfYzSIhXoydoHfOVsYJK5BxNKqH8HMxHWxizn5CbUs+kYY6aWfh5iPFctofb77v3cicgnAhMJTUTeETUReasmIotIW6UEuohI6iXVwt25T6a2IgSwU7mWoW3bOTECO4UxKhya0irdXQOvOE6rdHcNvKSPM8NbBGZq4DPCzp07cz/9dFufHj0mVVVUbM9duvTL3qNGjdrXAarQU6m1Wri3ZgV6Mtz16P7jXO4E9CMUHDqR0ESfizBrrrsBolSvR9cRE+jJPE8dpj3knrDrCjDBoRMwLSSDQDmwy1kXsJhQcKjdLtkgIpIGyYyb1MI9hd+hnUnE0YnwWBOJC8PGTE1ulR61tFSTW6Xbtp2VkhNunzJ+zLR69erOnXbt6jx20KBSjzPWLszNDR7bs2fl26tW9dy/f/+2dtzKvaW0xHf1dLeIb+7+azHrcm8M21cvQmOmIZjv7QA9MeMnd8y0HfP7LVUyJYEeLdFE5P6Y5bPCJyKvJ3Ii8i5NRBaRtkAJdBGR1FML9+TEPf9GWqU3qHggfqv0OpwADg1bpYevJ96SrdIz8X3XwCfMBx981GPXru4Fhx12xO7q6n4169Z926OdV6FL66nBtMNb71y2MGt3n4dJ8A4Hxjq37SWyheEOkmudG0+mJtBjqXS27c7lLExwqDdwKKEWkvt8Pt9G4BtCwaGdXq9XAWARyVRq4d64RGOm8FbpDZaVijWRGMiP8xxV4clvy7K2EadVuvP/2jS3Ss/U913jJsDv93s8waCV4/FEvB652dlBams9dXV17fFzka5jbq3PTHt6D2xCyzR95lxXAPwYk2zvjFnLOxvwYxK+mzHdvbZgvuc3V0u9Tm0tgR5LoonIo4HxhCYil8SYiFzR3IMWEWkuJdBFRFJPLdzDhLVKj64Kzw0Gg0MqKyu70DCwE2sNPLdVuhvYqUjQKt39t67lzrRJOvz7LkCM99mtPu/efVK1x5NFQUEPPwyjA1Shp/K4G/v5aM8V6Onev42pUAf4NyY4UUio2mIAcDgmwVuHWQcwPKle3YTnUgI9vgChbgGuPExw6GjgKsx7sBjY7fP51mAqZLYAW7xeb3mKjkNEpK1rdMzkJGkzYgJqrFbpgUBgIOCpqqr6XnR3LZreKj1iAnHYmKnasqy2+h20w7/v0vA9Hjx4cMWKLl2qvt65s/Dovn0rAIK2zaqSkoK8o47a1b17d3/LH2ZKdJQxk6utVqAnoxLzu3Ib8A/Md/O+hMZMx2I6SwHsJnLMVNKEY+uoFeipKjyJNxH5IMxyZWOA94FtPp+vmMiJyDs0EVlE0k0JdBGR1AvSgVu4x2mVHtHqL8lW6bZt270AwlqlN6gMDwvs1LThwE5TdIRzaK6MD4B98MFHPbZtyyvq3r2qauvWT4sALCvHXrfObq9V6Bn/njZRSySEo5+jAljtbGC+/3sJBYeOA05ybttFZHBodyPP05ES6J40P0+1s+0EjneerxSzrv13CU1q2Ofz+bYAawitC+jzer3tNVAsIpJI0uMm27bb1bjJmUSc74yXIqrCY1WIYya8xWqVHgSsYDA4kMSt0sP/3xESChozZYaY73Pfvn1rDznxRN/7//73wG1r1/bo3qlTXfH+/bm+3r0rJ5566vZYj5EW1xJroLfk+uQBzOTircBHznVdgYFEJtUtzHf6LZgK9c3OY+IVb3S0BLr7tzhdzxU+EdkGzsCMn2zgKEylutsevtSpUt9IaCJyqpctE5EMpwS6iEjqJVuB3uqVFEm2So9eAy/W3w4/oerwijit0iMCOxUVFTdkZWWtys/PX9CS59wGtPr73goyOQAWoa4u6Bk+vG43LIu43rJMq8JWOiwJacmEbbrFOwc/oQS5qzuhwNBATFIdzJp07n03YSo03ERuR0ugW6SukiLZ59pH5Nr0uZgq9WGYAJ2Nqcgo9fl832Ja9bsVF2VaF1BEOoBkf4+16vfnOK3So7trJdMqHSI7alUC28MS4NETiCtra2uP9vv95xQWFj7WUufbRqiFe4Y763vf2/Fp7941q1es6L52795O3QcO3H7W2LF7Bg0aVNXaxyb1OsLnNdE57AX+42xgvqv3IzRumoBZP93GVFCHT0R2J8Z3tAR6S4/PwHQKKMEk0sEk8Ttjxk2nETkReSsNJyK31c6UItIOKIEuIpJ6SQ32nbW2UxYQcAI7efGCOnECO8m0Sq+0LKuMUNv0lLRKd6rJMy0g4srU8854F130/W2YJKQ0X0sFB9KhNSqQaiEVAAAgAElEQVTQk1HqbF86lzsB/QkFh07CBIyCgA8TkLCc+6VTR06gxzqnWkxwqCTsfoWY4NBJwCRMVcY+YLvP51tFKKG+1ev11qT5uEVEUi3Z3++pHjc1aJUeZ7wUPok41kTHWiLHSNGt0sMnEFc0p1W6e3/btt129hkh7Fwz56QlgsfjYcyYMWVjxowpa/ze7UJLf5bTPWbqKBXoTVELbHA29/EHERozDcW0HAfzfX0zZjKVx9nSOdboyAn06Nct0UTkocAxznUVQFmMicilmogsIslSAl1EJPVS0sK9kVbp9clxGm+VXoOpCncDONGt0usrHlq4VXqmJtAz+Yt6Jr7fmSLVn+vW+jlp72ughz/HgagB1jmbu8/ehFoYjsD8/ToN00bPrVDfjKkMSFVwKN0tAsO15MQMD6FK/kTc9oT7CU28ycGsC+i+DxamU8Ben8+3HlhLKDi0S8EhEWnjkv17EXfcFNUqvcEE4jjLS+XG2FWAhpOISyzLqqB1W6WHT87W7/TMoDFTx9XRfobb8xroqXgOGzP22Qksd64rJDQReaDzfwv4FabVuztu2oL5Dp8q6V6OytXSS2wl+1yJJiKfCEwkNBF5hzMR2a1S3+b1eqtTd9gi0pEogS4iknpuC/f6II9t256owE5hIBAYCFBVVXVWnMBOrDXwwlulV1qWVd5Yq3Sn0r0tyuQEekaddwdZu15iS9dnOZklMCS2dAQ1bGCHs7nrD8zABIrqMAGiIzB/9+owgQi3feEWzDqBzdHSFQ4t9bk6kGr3OmCPs7kKMMGh0cB4IFhdXb1/7NixY3ft2jXStm21LRSRtiri9268VumAJxAIDKusrOxFWFW4829zWqVHTyCutCyrto1WeGdyAj3jxk1k3nssBy7eZ6a9V6C3hP+fvTcPj6O80r7vql7Uam3WYkmWrMXGRsa2vGAwYBuDcQyGMCQhE5JJwpUMQzIkJHmTfJMhyZs3GfLmmyEJJGGGSWYg4WNJnMA4ELZgDMbsNsbGu2xsY8nWYu2ypFart6rn+6OqXC25pW61up6u5fyuqy5brVY99aiqS3XOee77GPE7GoFiIf6B+vUGKL27d0KJmZZDKegCSrE33va9F+nDW4HOI884nVgw0UJkN5SYaTaABnX/oz/4wQ9qfv/73z8cDocfm+bxEgRhQ6iAThAEkXnkaDRaFYlErmCMeTCxVbryZlmei/Ot0s9L6oiiGAQQNWliJx2cmBDRcOq8CSIVsvn5sJMCnYdi4wyA99Wv3QCqoFsYXgJgrfq9boxNDvUjNexcQM/kWEF161S/dkUikRk9PT0Lwc+WniAIIh3k0dHRy2VZvjJuEbEr0RsZY5UActQYaUAQhIms0oOCIIzaaAGnU3uBazhx3k6cMzF1zHCdOF2BnipRADvUDQBmQHf2qgGwDLqrVBt0Z68O9WdTgVc7KjNYuKdLDAkWIjc3Ny+ORCKzMjQGQRA2gwroBEEQmYcJghARRfEsgL6JEjuRSOTCWCz2cb/f/9+c7P/MhlML6HZJ5qWDE883kR52/5zwapPBc4wYlGTP6bjXSqAnhmoArFBfD2JsQb0DiS3NeVsE8rRwN3IsKRqNaj0BqYBOEISZYWpsNBivFlf7hWv/Hx0ZGfmW2+3e6fP53sz2AWcBJxfQ7f48mAgnztkp2O0zzGM+doiZEhXpz6rbAfXrHADV0GOmK6G0GpGhLJCNj5uGkBheiwF4ttjiMVYwFAqFoNi7EwRBnAcV0AmCIDJMQUEBGx4e7na73a9DUeclJE4VYbdAKlWcXEB32rwpEWR/eCp3jRyP1/6NhOcYyX5P2gr//erXPug9AWsAXAWlXYkMpYgenxwKTGGcTMBLtcFlrGg0KgJgjDG6/xIEYWaYz+c7AOBwsvfBec/PGk6OGRljzInzJuwLz+cyHgVVq1vE8xoj2e8pDOCkumk/Uw49ZroQwGXq94agK9RbobTYklMcJxNkQ4Fu6FiSJPGMAwmCsBhUQCcIgjAG7QE22XuQwvvsipMTYY6ct4MSYE4qWDnlnGYKu1i4pztGCMAJdQMUVYGWHKqF0kf9CvV7ZwH0qP8vBtCexnhTwcoW7ucRjUYFQRAoEUQQhNlJ9T6VSmxlVxigPEfbqJXXVHDipJ04Z8J6GH2d2sXCPZ3fE4NSGO8CsFt9LR/6QuRaABdBaXkShRInzYBSXPdBibmMwsoW7gmRZVkAKdAJgpgAKqATBEEYQypJHi0ZIjo0GeLUArqTiqsalptzLBYTHn308bpLLrm4b+nSpcPZPh4HktL908Bx7aDWsMIYmi1hJ4D31NcKoKst5qqv3QzgRig9ATW1RRsUtUam4FlAN9wuXlWgUyKIIAizk+q9kDHGxORvsyWOVqBn+wCygGXmHIlEhG3bts38cN++stjoqHvmhRcOXL1uXXd1dXUmn8+I6WHlmIbH/q1UpA8AOKpugFLTmQU9biqAUkS/C8oi5Hhnr74MjK/Bu8WW4WOpCnSKmwiCSAgV0AmCIIwh6QMeWbg705JPPe+Om7fV2LlzZ/Hbbw/X9PW9mdvY2HhEFJ2aszUdVv/s8Dx+KxTQEzEMoEndZgH4MoBnAeRBSQ6thGL9DgDdGGthODCNcXkr0A1VUsRiMVKgEwRhBVIuoMP6zwBp4fCY0bHn3ezIsozHfve7OZEdO2oWuVyy3+ORj504UfrHQ4fKPn3nnUdqamqMVMBaFaOu5YnuozwU4lbev4ZVCujjiUGPgQDgCwAiUFqi1EKJmy5Wvzca997TUFpnxdIcl4sqfNxYhp4jUqATBDEZVEAnCIIwBglk4Z4MpyZEnDpvwCLzjsViwmuvvT8rHJ7tPnasrXT37t0zVq5ceTbbx2VyMhnU8ixkJhobBo7PUx1uJLyUB9o4Z6Co1DVKoast6gFcor4+grFqiw6kngyxm4W7COrlRxCE+SEL9+RQAd15mH7O+/btKwzs3l11S3l5oL6oKAwAl0mSsOnYsbLXtm2ruPWLXzyV7WM0KZmOmbKN1dsrmaUHeqbGCQM4oG4AkAPd9r0GwJUAvFD+pp7B2LgpVdc92/VAl2WZnLsIgpgQKqATBEEYQ8oW7tBtiZwGg3PnboZglyeWsSIEFPX5yZNiyZw5G/rb298ufO21XbMuueSSs6RCTwjva5lnwG5V7FKkjx9n/Fz61G2f+nUuxiaHrgbggZII6cDY5NDIBGMZbqvOcyxVgU6JIIIgzM5UFOhOfRBzcgEdcN68LfGMe/LkyfyZoZCnvqjo3CJjr8vFGgoLQ9uPHCkFQAX07GN04dYOn00rWbinM04YwIfqpr2nAnrMtADA5er3BqEr1FuhOH0lWuTGxVZ93FiGLgomC3eCICaDCugEQRDGkMoDntYD3Q6BRzo4VVFgiaSIU9HU58BC5OaWxmbNumz42LFjpEJ3DnZSoFt9jKmMMwrguLoBSrJFSw7VAlgMYJX6vQGMtTDsgf73iJdi2/CxSIFOEIRFSOk+5fAWSE4uoFPcZFLcbjcbAiAzBlHQL82wJIkur5cKUc7BDgp0O4yR6jgMiqtXJ4D31NcKMHYh8kIosVQUQBv0uKkNQAj2VKBTAZ0giAmhAjpBEIQxpKJAd7qFu1OtGLWFExAEx03f9BPW1OdVVZcNAUBeXkVUki4USIVuOqyaUOWZcLDDGOn+vjRbwjMAdqmvFUJPDNUAaIRuddgGIB9KAskLpX+gkRieSCMFOkEQFiHVe6FT4wZAjRkduujaqQsnTD/nJUuWnH1669bRdzs6Cq6orh4GgO5g0H0oGPTWLV/ek+3jMynZOK9WVqDzKjzbZYx0f1/DAI6oG6DUiqqgx0yXAFirfq8bQFD9fzGUhchGQgV0giCyDhXQCYIgjCGVYJ8s3C2QHDCQbPZ55o1l5vnGG3srh4b8Pkl6/dznMhoNuo4fl0v37t1btGLFisFsHp+J4dk7msf+LXPNJsCJCvRUGAJwWN0AxeK9CopCvQaAH8BFUKwMuzHWwjDT7hOGW7irVoSkQCcIwuyQhXtynKxAB5w3b0s8g86dO3e04YYbml974YW5TUePzvQLAtrdbsm1fHnHhg0burJ9fCbGiPM70T55fHYscb1Ogt16oGdinBiUGOh03Gsl0J295quvfRVKMT2+VVaH+vOZgouFu1pAp7iJIIiEUAGdIAjCGFJZvej0ZIhTC+hWDzKng+nP9+LFdf11dcPDSgswHUEoQ1lZWThLh2VaDFJDmf46mQZ26U9uxQL6eKJQ+nNqPTrvhKJEPw0lQVQPRXEBAAGMTQ6dwfRUCjwU6KIgCJQIIgjC7KR6n3Jq3AA4O2Z0XNykOpRZ4lzfeOONXfPnzw8cPHiwKByJuC6urR1ZuXLlgNfrnep5s8R8zYYJXCnsoEC3yxhGj9OvbvsBLAdwE4A/QFepXwVlcbIMpYgeHzcFpjEuKdAJgsg6VEAnCIIwhqQ2g3GJ7WwHPtnCqYmw+CSYU5JClpnnTTfd1JnB3Tnx+s4UyZQUVu1Rbhd1uB0K6InGCgLYq26AokqP7wl4DZT4SYKSHNIU6m0ARqY4lqHF7VgsRokggiCsQKrxgAxSoDvxudKJ8aJl4iYAaGhoGGloaJjKMxDBD6PzDVbfv53G4D0OAJxQN0D5+1wO3dlrIYAr1O+dhV5MPw3F6SvV4+Ry/5dlWQTFTQRBTAAV0AmCIIwhlT59Wi9sJyeDnJYQicfJczeEgYEBtyiKKCoqyqRtGJEdsv35sFTychxUQE+PRLbqQQDH1E17TyX0gvoSAKvV7/VjrNqiJ8H+NLgo0EGJIIIgzI+M1O6HzARqy6wgCIKTC+iAc+dtGCdOnPD39vZ6a2pqRqurq7PtsGXlZ+6pYsdr2ernj1cBnQe8CuiJYiYZQKe67VJfK4AeM9UAWKT+bATK4mMtZmoDMNF9iIu1OinQCYKYDCqgEwRBGEPKBfQU3mdL1GSQpeZ+8ODBgm3bts36xje+cUwU0173YPUgczoYdr5jsZhw772/XuT1uuUf/eiug0aNQ0wI7+vaaIW4UfAsCNupgM7DijyVBI1mS9gB4F31tSKMTQ4tUfcVxtiCejuUhBHAqQc6WbgTBGEBUo0HLBc3ZBBLxYzd3d3eP/3pT3XHjh0rzc/Pj1x//fWnrrrqqr40d+fUuMmQc93b2+v53W9+03D2wIFZ7lDIE8nLC1dfdlnrbV/60of5+flUPOKDna5pO1i4g8MYPJXhvGKmVOYzDKBJ3QDF4r0Kesy0Eor1O6Co0jVnr1YAA+rrhsdMABXQCYKYHCqgEwRBGEMqD65k4W6hucuyjAceeLDxwIG+2gsv3DJ4ww03dKW5K0slwTKE4UHP22+/XXL0qFThckXZzp07Wy+//PKzRo9JGEoyC3ej4FUYph7o5hpHGyudcQbV7ZD6tRdKckizMLwCwDp1311QkkJeKEkkwyAFOkEQFiHV+67jLdytoMA/ceKE/3tf//p6T0tLZSNjQheAX7z88qI9n/vcrm9/+9sfpLFLS8WLGcKwZ56H/uM/LvLt3Fn/xbKyobqSkpEPBgd9f926teExt1v+6te/fiL5HgiLwaNwa/X984qbzBzL8BonCuCUummUQomXagHUA7hEfX0ESsyknR8XDIxryMKdIIjJoAI6QRCEMUxFge7kZJBlEiJvvvlm6f79/XVDQ7Pznnzy2YaNGzd2TUOFDlho7mYnFosJW7e+XSNJC8VYLCK89NLrNVRA54ZR17FdPx92KW5TAX1iIgBa1E3bbxl0tcVcALkALodiZRivUu9EhpI3sVhMEASBEkEEQZidVNVyloobMoxlFt/++y9/uaTi5Mmqf6uo6KvweCSZMTzW21v0+B//eOnGjRvbFy5cGEhjt6aftxVoamrKHz50qPLT5eWDF82YEQKAlTNnjkRkWdy8c2dN32c/e6q0tDSa7eMk0iJbqnZeC5uNHsNOCnSrjdOnbvvUr3MBzMZYdy8XgO9BcQCLj5tGMnQM2gI1ipsIgkiIU4s2BEEQRkMW7smxTCJMlmVs2rS5YXS0Iaei4st9TU0js7ds2VKR5u7sZNs2VQw534r6PFZeUbF2qLx87dDhw6OVO3funGHEWAQXJrtOjC6oWn3/vMcwGisW0MfDoPRDfx/AMwAegJLwOQjgAIB8AOsB3A7guwD+HsBHAFwIwJ/uoJIkieBj40gQBDEdUo2HLBM3GIAlYsZQKCQ279kz50a/f7TC45EAQBQE/F1p6WBJIJC7ZcuW6jR269S4KePnurOz0+cOhTxzCwrG9Bqek58fcgeD3q6urpxMj0mMxQAXiWT7s7oC3S7FbSsWtpONY1SMMQrgOIBXATwKYDuUxckvAxgCsBjApwH8E4BvAPgEFNV6OaZx3yQFOkEQk0EKdIIgCGNI+kAZ15vU1MkQA7FMIkxTn8+Y8eXh/PzG8MDAksJpqNDP2TAKgiWmnwkMC+Ti1ef5+TURAOjuvrCQVOh8EQQh0+fY7glTq8+PFOjTH6sbwFvq1y4AldCVFksBrFa/14exaoveVI5TkiRSUhAEYQVkpHbvdbyFO0weN8ViMQGyLPpFcUwc7BEEeHCutchUYVawrs8whsTIVVVVo7Hc3OiJoSHfouLiUe31k8PDvlheXqSioiI82c8TGcPqMUA8PD6b1APdueNoyADeVTcAKMRYhXqjekxhAG3QY6Y2KMX35ANQD3SCICaBCugEQRDGICFFBTpjzLHJICskROLV52VllwwCQHHxJ4aamu6evWXLlopp9EI3/dwzhZELBTT1eXHxspFweNAFADNmLB85fPhYJfVCtzVWVYjztHDngZ0K6CKncYDzE08SgHZ126m+NgNjk0NL1Z8LYWxBvR1KT8ExqAV0UqATBGF2GPSC4WT3YMssvDUASxTQ8/PzpfKGhvaXdu26cF1h4UiOush4+/Cwv9fnC39x1SqKmVIn43NesGDBSOGSJR1/efvtuRFZFmrz8sIfDA3lvhwI5NZfffURsm8n0sTqjle87i92KmzzjpnGxzNDAA6rGwB4AFRDj5kuB3A1lGPshhIvnVb/TZgbUvOSFDcRBJEQKqATBEEYQypJHkskQwzEEomwt956q2T//v46YD56ev5cpL0+PFyan44K3QClrpXI+Pl+//19pYLA2ODgptzBQeRqr7tcAnv//fdLs1RAd9I55v0ZNv09Iwm87NXtUtjmrUDnlThJJfF0Vt0Oql97oSeHaqEo1HPU/XRCLajLstwqiuKgqvQjJQVBEGaHLNyTY5mY8Qtf+tKB+06erPjGmTMzL/d6I12xmPtNUWRz1q8/uGrVqv40dumkZ2oNw+b8pTvv/OBht1t+bP/+Kk93d1HE7w/Xbtx49At///cnjRqTyCpGxwQ8eqDbQR1uN2U4TwV6KjFTFECLugHK8ZVCL6jPgWLzDgABxC1ElmX5jCiKElm4EwQxGVRAJwiCMIZUHr7Iwt0Ccx8dHXVXVroHJGmPCOw593p5OQZF0c/StXCHBeaeQQwLsD73ub9rWbu2NaGipa6uLmjUuIRt4VmwNQoqoKc/VjbVFMmIAGhWN20fM6Enh+YBWLl7927ccccdrKGhYXTmzJnDgiBcAmA/Y4yUZQRBmJGULNzV1leOdO2KW3xr+thh7dq1/bm/+tWWPzz+eMNTH3xQmVNYOLpu3bqTt99++8k0YibAIvGiVSgtLY1+5/vfbzp16tTJ3t7enOrq6tHKysqULI6JjGDHa9nKMRNABXQzj5PuWAxKy6teAHvV1/wAZkOPm64B4L7lllvY6Oho5IILLmD9/f1XCIJwhDHWk6FjJwjCJlABnSAIwhhkpK5Ad2QyCBZJiFx33XXd11133RYDdm36uRtAxudcXl4eKS8vp8SPc7B6gZvX8dulsG3nAvp0x9JsCbuhr+7K83g8C9evX7/04MGD5e3t7eUA3gMwKgjCLgC/Z4z9dprjEgRBZJJ4C/dk73PiszNgscW3l1566eCll166K4O7tMS8M4yhc66rqwvV1dWFjByD4MpEz5SkQE8NKqCbc5xMjhUEcEzdAMAFoGLVqlWr9u3bV7Nr164CAPcAuEcQhOMA3gHwT4yx3gyMTRCExaECOkEQhDEkTfJYSU1gEE5NhDnxvFu12EmkTsbOsdqDLFvw6oFOFu7mGkcby0x2hOkwsnz58veWL1/+3m9+85v5H3zwwbxwOPy/AKyCYvleOp2dC4Lw/wH4wiRvuZwxlsmiCUEQ9mcqBXQnLzrO9vNRtnBivMgYo9DJxmTy5Jrhs2F0TGN0eyVefdbtFMvwbntlxFgSgI5vf/vbmwGgoaHhdgB3AMiHEjddCqXXetpQ3EQQ9oEK6ARBEMaQsoU7Y8zJySAzBH28oYwIYSec+Bk2O7wSQQAV0KczlqGJJ0mSREEQYoyxnQB2AvhFBnb7YwC/SfD681CUHe9lYAyCIJxFqvdCmTHm1PyVExffalDcRBDmxQ73JLJwN+84PMcSAXQwxt4H8KcM7ZPiJoKwCU4NQAiCIIxmKhbudgg80sHpBXTHzd2hyhkis1hdIU4KdHOOAxinCk+E4edIkiQBqS3mSxnGWHwPdgCAIAhXASgD8GOSzBEEkQZk4Z4cx8YOcO55d+KcnQDv88ojJrBDTGOHMXiOY6uYCQBkWaa4iSCICXGq6pEgCMJoJFABPRlOTYgAcFwxmYIDYqrY/ZqxcjILsGcBnacdIY8CuogMJ4Im4B+g/N4e5jAWQRD2QwYV0JPh2JhREATAefO2+zMwYR94fDapB7pzx9HGMjw+U11BKW4iCCIhVEAnCIIwhlQU6NqDoFPvxU5NhFFShLAjmb6uJ7o3WF0hbpc+e7zu3drfRzslg7gsCjBCSTEeQRAKAXwSwCuMsdNGjkUQhG1JtTgsw9kxE0Bxk5Nw4rl2CkZc0xPt0w4KdKMhBbp5xwE4qd15FNApbiII6+LUAIQgCMJokq6SFATByckQbf5OnLuTz7sT52x3jDindr5OeFm4G41dFeg8C+hG90A3vIAO4LMAfAB+Z/A4BEHYF83CPZX32fn5YDKcHDs4+bw7CTrH6ZHt35vR4/Ms1BqJ3QrbtuuBrrpDGq10p7iJICwKFdAJgiCMgXqgJ8fpCREnzd0Oge+UcZhNf6bJ1jXDq2Br9X6BGnYqoPMah8ucOFm4/wOAPgB/MXgcgiDsC1m4J8Hpi67hvHk7Mm5yCHa8lq0eMxk9hoadepPzbntlFwt3ipsIwqJQAZ0gCMIYyMI9OU5NhDk5KeLE802kR7Ys3I3GTlaE4DiO0fC8rrjY0hutQBcEoRHACgCPM8aiRo1DEITtIQv35Di5gG7V573p4phzHbdAhMg8RscEPBToPPZPFu7mHAewiYU7xU0EYW3c2T4AgiAIm6KpKSaDAY5WqZqqgB6JRITHHnus/sYbb2yvrKyMGDiUE5NglBixP3Y5x3ZJ1NipgG6borYKLwt3kTFmpJLidii/L7IhJAhiOqQSMwEmixs4Y5qYUZZlbN++vWznzp0VsiwLy5Yt673++uu73G63UX8/nXje7fJMTSTGbufXDq5adolneMZNdrNwN1qBTnETQVgYKqATBEEYQ6qJIMB5SQENUyVEnn322Vl/+tPupX19/b677vrnIxyGNM3cCWIaGHUdp7QAiUiI3Xqg26aoPW4sHgp0QxQOgiB4AXwOwLuMsSYjxiAIwjFoPdAn/dslCAIzQwE5S5giZpRlGf/37ruXtLzyykX1oZDXA+AvL7wQe/2VVz78yU9/utvn8xn1N9Sp550gMoGVFehGw9Ppyk4FdN490C1t4U5xE0FYH6daYBEEQRhN0gdKQRDIwt0kQVckEhGeffa1+X19FUVvvHFibktLS65RYzncps4U55uwNDwU4lbvtWc3C3fb9CVX4aJ2l2XZSCXFJwCUAPitQfsnCMI5kIV7ckxRQP/rX/9acerlly+6Mycn+q91dd3/t76++3uFhcHAjh3z/vCHP9QaNKxT4yaKmewJ74XHPK4jK8c0dmxJZbcCuh0s3CluIgiL49QAhCAIwmhSWSVpimRIFjFNAf3ZZ5+d1dwslNfX39HV11dR+MQTT9YbOJwTz7tTk19Eetj5s0EF9KmPY7cCOi8LdyN7oN8GYBjAEwbtnyAI50AW7skxReywc8eOWQtCIe+VxcUjoqAcSmNBQehSxoR9O3bMNmhYJ553ipuIVMn2Z4MWNqc+jp0K21xU4XFjWb2ATnETQVgcKqATBEEYA1m4J8cUCRFNfc7YSsHvnxMpKNgwarQKXSXrcyfSo7Oz03vixAl/to/DZGQssGVs0l0ZnejguaLeSKiAPvVxYKexJEkSAcSM2Ddj7DrGWBFjbMSI/RME4ShSsnBP8T22JM69Kqvzj4bDrvwETlp5LpcsRSJGtod05Hm3OrIsY9euXTOefvrpWTt27CiWZV71NtOT0ZgpSWsLHs/RFDclx44FdOqBnvq+KW4iCItDPdAJgiCMIZUI0fEW7mboZaipz8vLbxgEgLKyq4abm18uf+KJJ+sN6oXOgKRFQruS9fM9XWRZxr33/mfj4GDId//9d+/w+/2Ozgal+hlmjLllWc6XZTmPMZbHGMtX/z33fwB5jLF8ALlQejcvBtCsbj2wR4KGFOhTH8c2RW0VLhbujDGBMebo+xNBEJZAU6CThfvEmKKAvmDRor5dr7/OOkIhd5XPFwOAwWhU3B2Nuuc0NnYaNKwdnv3SwdIxU2dnp/dX99yzLNjUNMszOuqN+XyRpxsaur5+1137ampqQtk+PjOjPr/5tZhJluX4mCkPQH78/wG4ACwFcBJKzHQaQJjT4dpFgW40dits87Zw5xHP8BqHIAgLQgV0giAIY9DUFBMiCOee1y0dIE+DrCtJIpGI8Nxzr88LBmf5hoYORoeGDuYCAGOVwhtv7J/76U+3tNTX149meFhTJMF4Yqe+72+++WZpU6u3HacAACAASURBVFN4VjTqcb/wwguzPvWpT7Vn+5iygap48EmSVAwAkiTVhEKhIrUAnhdfHAeQB8CbYDdBQRBGAIwIghAQBKFLEIQRWZZnyLLsU39uA5TEUBBACwAtQWtVBTqvArrR2C3hxKWorcLNwp0xZpQVIUEQRKZI9b5rioW3WcIUscPNN9/c9u727W0/Pny4drXXG/WIIntndDSnf+7cM1+/5ZYWg4Z14nm3fNz061/9alH+7t31X6moONtQVXX2eCCQ84d9+2p//ctfxv7tF7/Yne3j4w1jzKUWxP2MsdxwOLxs/IJiqIVxAH6c/1mPAQgIgjCibp2iKI4AkCVJmgnlmlkM4Ar1/x1Qiun5sLYC3Q5xmTYOr881r7iJFOgEQTgGKqATBEEYQ8rJIDiokDqOrM99ZGTE5XKB1dWd7lIWaysUFiLo84mxvr4+rwEFdA2nnnfLIssynnvulfpI5CK3210ib936Vv1HP/rRMwlU6JZMfDHGRFmW/ara4TylOMYqHvKgFLYBAJIkrYFSEBxRi+IBQRD6RVFsVYvjI4IgjIiiGFD/DQqCkLCAGA6HF8uyzAAchvKsWgNgDoB6AAvUt30SwIdQkkMtAPoN+JVYFVKgpzcOwEd5wM3CnTFmiIU7QRBEBiEL9+SYooBeVFQU+/G99+549NFHu156990aJsvinIsvbvvK5z9/sq6uzkhVsRPPu2XnfOLECX//gQPV/1hSMtxQUBAGgPn5+eFPlJUNPdDUNOvQoUP5ixcvDmT7OKeDupDYyxib1F0rLmbyxf98NBr9GIBQXMw0IghCX3zMFB83CYIQiRNfnEOSpBmhUGgBY2w/gFEAJdBjpuXq2IDSg1lz9WpD5lr8WPY6VbFTPMO7HZVtCuhqewkBVEAnCGICqIBOEARhDKkm4WV1taMTyXoirLi4OPbww//xJudhLVlczRCWDrI19Xl5+YYhr3eG1NKyq8zsKnTVOl0riE+U2IlXPIwnijjFgyiKHVCL5KIoBhhjcjQa/bTH49ns9XoPJ0ruTJMY9IQPAJQBuBOKTWEJgEVQrqsh6MX0ZgCDaY5nF6WDXVQOZOGeJrIsCzCoBzpBEEQGIQv35JiigA4A5eXlke985ztHARzlNKQT4yZLz7mnpyfHHQ576goLxyxCr/P7I+6hIX9vb28OANMV0FXr9FwtZoorjJ9z1BrnrjU+n86gLyQeEQRhUBTF9riCeCAaja5kjPny8vIeFQQhk8U67ZrpV7c96tefAzADyu/7UgBroRQJW6HHTO2YXuHQygp0DTuMYdcCuuHW6pFIRHu2oAI6QRAJoQI6QRCEMZACPTlOnbtpkmAcsXQiCBirPi8omBcGAJdrJZtEhW4IquIhJ74fnizL59mmxykechLsZnSc4qEnkeJBFMURQRCikx2PJEkF0WgUEykj0mSyoFwrCL4PpYieA6AOitJiDpQegABwFmML6sOZOrgMYfVkk50UG9o4sNNYqgKdEkEEQZidVJ+LnRo3AOrvyIFW5oCzz7slmTNnzkgkPz90YHAw9yM+37nn7wODg7kRvz80Z86cEV7Holqn+ydTikONoaDETYms088VwAVB6BZFsVmLm9RYSYuZRpO1LYvFYksAeDJYPE/22YhCWVT8pPrecugx0xUArlbf0wo9bupA6gVLHp9NOyxstlNspo3Fq1+44b+7aDSq/e4obiIIIiFUQCcIgjAGKqAnQQ0wHTl3FSfO3bJz1tTn+fkXhQKBFi8AFBTMH21u3jlzuip0VfHg1xI78cXxCRQPrnG7iLdOHxEEYUC1Tj+X8FETO1qih1fAy4MwgGPqBgC5UArq8faFANAHvZjeAmCi5B0vBbqR8BzDDgknbRxwHsvQz6EsywJZuBMEYQE0C/dJcXLcEFeUc+T84cx5W3bOlZWVkflr15589tlnGyNnzojzCwpCHwYCOVuCwZzajRubpmv3L8uyN67dVDJ3LV+CXYQxtp94X3zMFBcvjQiCEM6wu1Y2zysD0KVu76rHUgklZpoD4EoA6wFEAJyCHjd1YvJ7tJXjJl7nw44FdNtYuEejUVKgEwQxKVRAJwiCMIaULdzhbDtCyyYHpoETk2CWV6AfOHCgxOORJEnaLA4NIVd7PTcXkcOHj5Z86lM4r4DOGMuNRqNViRI74xQPfpx/PUQxVvHQqakcEigeQskUDzZkovmOQrEV1axF86AU0uuhJIdWqK/3QC+mt6g/xwMeyQ07JWl4JWi42KqrcPndybIsghJBBEGYH7JwT44TYwcNJ8aLln+mv/PrXz/2YE6O9OfXXpsjBgK5st8funDjxg++9OUvnxj/XtVdS4zFYjMTFcahxEvx7lqe8bsAEIxz1xoSBOFMfEF8nLsWPRspMABn1O0dKPfXKugx0zoAGwCEoBTUtbipG/zuSXZprWWn2AxQrhWeBXRDP7OqAp0xxix/7yUIwhiogE4QBGEMpEBPjlPnTg/mFuS222778Npr23oFQfCrW54gCH4A/pKSkrpgMHhRvOoBAGKx2EdjsTEC0JAgCAHohfGeiRQPoihGsjDN6ZDp63qi/U31njEC4LC6AUAB9MTQfAAr1dc7oSSFciYZO5NQAT31cWxT1FbhUqyXJEmQZZkU6ARBmB2ycE+O0wvohMXweDzCV77yla7ArbcGu7u7i0tLS71+v7+MMVYXDAbz4wvjUK3TQ6HQV+N2IUGJl861m1Kt088Vw+PipqDFFhLzPNapPEfLANrU7S0ojmfV0F29NqivjUJXp/N4TrdLAd1o7KpAF6G3cDOEaDQq2syhjyCIDEMFdIIgCGNI9QHM6ckgx83d4TaMGZtzLBYTDhw4ULBs2bIhUUxPkKRap+dqlunje+NhnOKhurp6/HMTg1oMZ4wFBEE4K4piO4CQJEnrXC7Xmx6P54iqFA/aVPFgtet4GMBBdQOAIugF9YUACtXXvwQ9OXQaip1hJuD1+7JTAZ1HQoOLrTrPsdReuXa85xAEYS80BXoyHBk3qDg5dnDqec/YnDs6OnIOHz5c6HK52NKlSwdLS0uj6eyHMeaRZfm8VlMTuGvlAoDL5cKsWbMAALFYLIyx7loDgiCMSJI0hzGWn5OT80ycu1amrdPtjhHP4xKUGOg0gNeh1A9mQ7d836i+bwOUWKoFStzUn8FjsFPcxCs24xXL2MbCPRaLUcxEEMSkUAGdIAjCGKay0tfJdoROjoydNPeMBz2vv/566dNP77zgM58JHl+zZs25QJ0x5orrJZ6wNx50xUMi6/QYxvbF69JUDgkUD6OJFA+yLHuDweA6URS73G73mUzP3eFkunA7CGC/ugFK77/LofRMXwJgFZT7dAd068JWKBb76cDLwt1oSIFu8rEkSeK1+IAgCGI6aD3QycJ9YpxcQAecN++MPR88+eST1ce2bq33Dw76mSCwt8vKRi77+MdPXHvttT2qdXpuXNyUsBge56413jodUKzTNXet4Tjr9DHuWmrclFBFOjo6mi/Lssfj8ZzO1LxNTKav5WT7y+RzdAx666vtUK6H70OJi4oBLFLHG4YSM2lx09lpjMnLwt1o7OQOpo3Fs4BuaDwjSZJgMRcLgiA4QwV0giAIYyAL9+Q4de70cD4F1OSONz6xE41G8w4caJ4XCFSVNDU1Ny5ZskRyuVxagseXYDchtS+elszpS9QXT/1/ZDqKh0Ag4Hr44Ucv+NjHbkRlZWXa+yGyxiiUBNFT6tdlGNs//UooK9TboCst2mCuVet2StLYsYDOzcId5rouCYIgEkEW7slxcgHdqXFTWueaMSbKsuxnjOXv2bOn8szrrzf8TVGRa9FFFwmSKOa839GRd3DHjiua588XZs6c6cf5i1I063QtTuoVRfFUgoJ4YDrW6YFAwPXiiy9WHtu7d2ZJaWnp3MWLvStWrPCUlZWlu0CV4I92rpoA7IXSBqsWuuX7EvX7Z6HHTM1QCuxmwU7xDO8COq8Yw/B+69FoVATFTARBTAIV0AmCIIwhZQt31WbViTg1EcaAc/a6TkMAxlinJ1I85AHIH6d4GPO8cuzYMdbd7Y1edNFVoRMnXvF8+OGH3QsWLGiPUzyMV4pzC4i2bt1a8cYbZ+urqt7H3/zNDU46x7wTnEaNN/6c9arbbvXrcugF9csAXAWl4N4KXWnRjon/BvBSoNuhl582jt0K6FwsFiVJAigZRBCE+SEL9yTELex04vwde941GGNuWZbzx7eamsBdK1f7uc62NizMz8fiefNikKSwR5bDK2trw6eamqKHDh06s2HDhg41ToqPmUJGW6eHQiHxgXvvXch2765d6nLJ/vLy3A+PHs3dsXXrkq9973sHHFBEt8uikPEXShjAcXUDlEXt9dDjpmXq6/3QY6ZmACNJxrBLD3S7FdBtZeFOPdAJgpgMKqATBEEYA1m4J8fpCRFbzV1VPCS0AGSMFQJAJBLZEIlErgeQh/PnL0G3Tg8IgtCtJnJG4vrhjcRisZE//3nbvNbWZcUez/yB9vZDpc8++2pPY2PjIbfbndWERCAQcG3ffqAmFJrr2b+/G4sXN/sWLFiQzUPigRHX8WT75PG5mew66la3XeqxVEBXWqwBcA0URcZp6EqLTujFUl7JDbskaXglaLiowlW4/O5kWSYFOkEQVoAs3FPDyXGTreatumv54ltNaf8HkCfL8hwA/kAg8A0oMZM3wW6C49y1usa7a7U0Nc1e1tGR53e7x/Sk9vf2lrW3tjb7fL4PjZ/t+Wzfvr0ssmfP7K/MmjU4Oy8vEqmoqLyEMd9/v/XWrC1btvR8/vOfd4KVe0ZQr6XJ3sLjOXqi/YcAHFU3QGmdVg89blqhvt4DvZjeAsUNTINXHGCHeIa3m5ZtLNxJgU4QRDKogE4QBGEMKSvQYbOkwBRw6twts+qcMeaZguIhkXV6GEpiJwgAgiD0uVyu1kS98QRBCKeieHj99dfLmptdpdXVVwwCQGXlFUPHjx8p27lzZ3F8L/RssHXr1orWVn/xvHm39AwO/hVvvPFOsQMK6BmHKZmgbH1OpnJPYlCK450AdkBJJsyCnhy6CsBHoHwOTkFJCiXs/5hheKoC7JBw0sYBp7F4WbhTMoggCCugKdCTFcedGjdoOHX+loib4qzT8+IL4xO4a+UBcI3bhQy1GA7lsxBzuVxNE7hrBVNRSxZWVXk+3LfvQkmW4RKVj1cwGhVbGBNqq6qCmf0NpM6xDz6YMU+WXbPz8iLaawU5OVjs9cb2HDxYBmURql2x02d4qnMJQrF7b1K/zoceM80DcKn6ehf0YrrRvy+e58NOBXSesabhxXpSoBMEkQwqoBMEQRgD9UBPjlPnnrU+huoqdc06/bzCOJRi+DkFBADP+F1grOJhSBCEMwn6iWtF8RgAyLLsDQaD33O73XtzcnIOp3v8kUhE2L79/epotN4ViQy6IpFBFwCEQlWebdverb788ssHsqVC19TnOTlrY15voVxUtAx7924uPnToUP7ixYsD2Tgmm2J0cmA6CQEZin17O4C3oQT81dCVFtdAf/b+GygWhy1QFO2ZhMe9xW4KdC626jzHIgt3giAsQkrPxWqvZSfGDRpObfuVtfOuWqcnKojnjy+MQ1HXjicCvZ/4iCiKHdrXWjE87v/nrNNHR0dvlGV5Vm5u7ivTOf6r1q3r2vT++5Wbjh8vv7isLBCTZezq68sfnjOn78orr+ydzr6ng8vtlqMJnu0isiwIbjc9t6THRM/Khqt3Jxk7GQEAh9QNAAqhx0wXAbhcfT0XyvNsM5TFFRFkDjvFM3YtoPOwcKdFxwRBTAoV0AmCIIyBLNyT4/REWEbmPs46PaFSHGphHEpRfPz1JkG3/xsRBKFXFMWW8YoHtSAeVBOYWaGtrc0XCIS8RUUfjo6MfHjOznDGDATPno36urq6vNXV1eFsHJumPq+rW9MHQCwurkNTU7X3pZderV68ePEH2TgmzmTyurDLfUGG0hu9FcAbUJ67LwewHopjw3VQPo9BjO0F2DfNce2UpLGjAp2XhTslgwiCsAKpus7IAATGWHxPcCfhyLhJPdeZipnAGMuJbzUly/K5OGl8YRyJrdNH1YXEWtzUE7+IWGs7pcZNWe3lPXfu3NGP3XHH4Zefe67m2MmTJUwUWfHq1S2fu/nm1tLS0qwdW2Nj48Cr27bFDg8M5C4qLh4FgO5AQDgky0LD8uU92ToujtglZsr02EMA9qsbABQDuA2KFXwjgFVQfnft0GOmViits9LFTvEMFdDThBToBEEkgwroBEEQxiAjhaDCyWoKB889aQCgWqenqnjITbALzTpd29o01XgCxUNK1ukZYloDzZ07d/T//J+v7otGo+ftJycnRy4uLuZhj30eoVBI3L79QE0wWJ5z+vRfSwRBcEWj+YhEmPj++/1VH3zwQVtDQ8NINo6NA0ZcPNm06zQySI9BV5s/AeVzWgPdvvAGdfwA9P7pLQAG0hjLLkkaOxbQeVm4A1RAJwjC/KTaAz1eqW4JW+8MQ3FTom8yJjDG/GrcNFGrqXh3rYms0zV3rX5RFLV2U/GtprR/eRVZMnKuGxsbhxsbG5v6+vo8brebFRUVZSVWimf16tX9B/buPfn466/PndPTU1AwMJDb4fe7XCtXnt64cWNXto+PmDJG3Y8HoMRKxwFsBVAKXaF+MYA1UD6/bdBjpjak1y7LDvEM7wI6r3uhaPRYpEAnCCIZVEAnCIIwBrJwT45j5q4qHnyyLOdLkjQLAGKx2MJYLDYPag/xcX3xEikegnGKh4AgCF2TKB6ynhyJJ5Oq9WwqJiajsbG6e+7c0QHgNERRdNfUzL60vz/aEQzObPP7/RSQTY3J7gs8kwNGEH/8UQAn1Q1QPve10JNDi9X3D0JXWrSoXycbwy5JGjsW0LlYuKtWv6SmIAjC1BQUFLDh4WEqoCfHSXGTS5ZlrdWUH4ArFAqtmcBdy4/zfy9RjHXX6lTjo/iCuOauFcqmu9YEZPx4zBQ/ud1uduc3vnH8rYsv7jtw4EBJTmlp4WUXXIAlS5Y0+Xw+uz+38P4MG3m/5NUySjv+PnXbrX49E0rMNAfASgBXQSmet0KPmzoweWGUZzxjNKRAT5NYLEYxE0EQk0IFdIIgCGMgC/fkWDoRpFqn+zX7v7ikznjVg6Z4GHOeJUlaCd3+T7NOPzWB4iFItlLmxefzyV/+8peata/Vnu/weDydOTk5x7N5bBYmW8lMXgmBRGNEAJxQN0Cxea+DrlBfqr4+gLGW74Fx+7FTkobX+eCiClfh8ruTJEkAqSkIgrAGqTzjau8xXI1mUiwbN8VZpydsNTW+MA4gZ/w+YrHYDEEQAtAdtnriFxFrcZPqrmWaYjGRGFEUsXbt2v61a9f2j46ONsiynO+A4rldMfrZeaL996jbLij3xgroMdMqAOugLKY5DT1mOoOxfz943VPttLgZUP4O26aALkkSWbgTBDEpVEAnCIIwhlQfwCybDMkADFCSKmbpZcgYc09kATiB4mE8EYxVPHRoX2vFcMaYKxKJ3Or1eh/zer2nuE4w+5jjRBuMCZUshsNxznZRoKdCCMAH6gYo7RrqMda+EFDUGPGW73ZK0vBWoPNInvDqgU4FdIIgrEIqra/iFehOxFQxo2qdnqvFTOPbTkFtNRXnrjU+98igW6cHBEE4K4pie7y7liiKgXA4/BEAUl5e3h85TzHbmOZcExnHiOe/ifZpBwV6KjAAneq2E0qBtxJ6zLQWwHoouZpT0GMmKe7njcROsZk2Fs/YnyzcCYLIKlRAJwiCMAaycE+O4VaMcdbp5wrg2v8Rl9iJUzwksk4fHad46JpA8TCSiuJBkqRiABDMsmqAD1YtdhLJsdt1bHRCYDrJjVEAR9QNUO5Z9VCSQ3MBXKK+HoJiYbgASnIolN6hTgqv885L4WC7HuhUQCcIwkKkXEBnjDnsEfochseMqnV6XoruWoms02MY666lWaefK4jHOWyNprL4MhKJRHF+33K7Q3ETYSWMjpvS2b8Mxb69A8DbUJ69q6Bbvl8DpR4SVt9/kfr/njTHSwYV0NPH8FiQXLsIgkgGFdAJgiCMIdVVkjJjzMkW7sAUH8BVxYM/rjfeeT3E45M8OD/pImOs4qFfFMXW8YoHDtbp3LN/e/bsKQKAFStWJOuhTBCEsWQiETAC4LC6AUABlKTQVQAKAXxafb0TutLiFPRk0XSwk+WhNg44j2WomkKWZVJTEARhFZLeq+Kex50cN03pb6+6kNiboCA+kbuWL8FuQmrMpC0c7otfRBwfNwmCEDFocQPXmCkYDIqbN2+e3d7enj937tyhT3ziE+1er5d3UduRq0QcQKYXAGXzOuE1diY+ezKANnV7E0p+aDaAxVAWIF8G4AoAQeh27y0AejMwNkAFdFOPFYvFREEQKGYiCGJCqIBOEARhDKRAT865AnqcdfqEvfEwVvEwnijiFA+iKGrW6RMpHnjNMRHnFDQ8Bw0Gg+Jdd/1knSAI7Jlnfv+c3+/PRp8np17rROYwOjnAS4FuBMMADgC4AMAMAE9DV6gvgpIYYlDUGC1QkkOnodw/pwrPwrZdC+ikQCcIglBIJR4iC3clZtKs0xMpxc8Vw+PctRJZpwfj3LUGBUFoj3fX0lTj6r/Z/jvCeMZMu3btmvF/vv3ta6WOjpkFsiy+4nZLv3/ooY57//M/X25oaBjhdBikQCcyBY/naCvGTRKURcUhKAX0R6EU1TWF+vVQFmsFoMdMzQAG0hzPjgV0XrkkHhbuPOdDEIQFoQI6QRCEMaSqrk7FstByqIqHnPH98OL/L8tyOQCMjIx8B0BOgt2MxqnERwRB6EmkeEjVOt1EZCUp8sgjj8xpaxPKBQHs8ccfr//Hf/zHkxyHz+icd+zYUVxbWxusrq7OhJKWyAyZPMe2uyfGwSO5of3dOQtgn7oBQAn0XoDLAKyG8jeoHbrSohWKBWsqUAE9PbhYuJMdIUEQFiKVeEhLbtvuGYExJmox0wSLifMA5MZisTWxWGwdElunx7eX6hZFsTm+1VRcYTwl63QTwe1YZVnG3d/97jUXtLZW/nDGjKG5Xm/scDjs+dHx47U//O531zzx9NMv8TqWTDA4OOjesmVL5eDgoHfBggVn16xZ0y+KTjVwMBVW+vxNBq8e6DwK9DEoi4qb1a+9AGoxdiGyAGAIeszUDGAqrn5Gn3cu8YWK3SzcybWLIIhJoQI6QRCEMchI7UGPwSJWhHHW6ckUD3kA8nG+dTqDav+n/jvCGCt1u91vC4IwlEDxYNdVoNwVNMFgUHzqqZcbgY8wxiRh8+aXGm+99daWLKjQpz3n9vb2nJ/85Der584t67n//nvfpURQ1jHqOk52/7SqAp0HE82hX932qF+XQVdaXApgLZTkQRt0pUU7EicU7KZA5510AsjCnSAIQiOV+6Hm4GSJBz9ZljXr9FTctRJZp4cR567FGJMEQehwu92HxyvFBUEI27gvPDe3tpdeeql8tLW14puFhYG5Xm8MABbl5ET/MS8v+P0jR+r2799fsHTp0mEex4JpzvnVV18te/BnP1vl6ews8UmS8KrPF3ny4oub/9+f/WxXUVFRqgslCXtgZQW60Uy0gDYC4IS6AYrYow76QuSl6usDGGv5PtH9wW4KdMOL2nHwsHAXTOC2QhCEiaECOkEQhDEw6AH/ZA98WbVwZ4y5VOv0MSqHcT3FtSSPP8GxRjFW8dCpqRziCuGahXooXvEQDocXRaPRWq/X+64oihGO0zYL3M67pj4vKfm7IcYktLW9Vs5ZhZ6xoGfTpk1zOjoKS4aGBnLfeeed42vWrOnP1L4J0zDZ9WL1DDGv5EYq++9Vt/egHFc5dKXF5QCuhq7IaIGSHOqArhS0UwGdS1F73FiGzosxBlABnSAIa5CKAj2rFu7qQmLfREpxqMVw7f8APON3AcU6XesnPiQIwpkJ3LWCgiCMKXIGAoFviaJ4KicnZzefGZsKLue8u7s71xWLueZ4PGOczeZ4PDH3yEhuV1eXDxMXyDLJtJ4PBgYG3P99zz2rL+3sLLmjoqK/1O2W3gsEcv/znXcW/PIXvwj8y913H8jUgRKmYaJrxsjPjh0U6BrJxggDOKZuAJALvaA+B8By9fU+6MX0FgBa2we7FdB590A3ND4jBTpBEMmgAjpBEIQxpGozmFELd9U63TuJBeCYwjgSW6eH4lXigiD0xvXFG98bL+3Cd1wx3eoFsanCdZV2vPrc650dAwDGrs6mCj1t2tvbc7ZtOzQ/P/8zo6Oj+3OeeOKp+atWrSIVehZRi3R2gpdVIA8L96nAAHSp27vqPiqhKy3WALgGiiLjNPQkA4/fF1m4p4GqQLfM/Z0gCEcjIQsW7nHW6XkTtZ2C6rAFpSg+/oFTguqupbWbUq3T4xcQawuKg9O0Ts/qousswu1B8+KLL+5/2OcLbw8Gcz9eUBDUXn8tGPTJeXmjy5Ytm4pdc9Z44YUXqnzd3cVfrajoL/V4JAC4rKBg9FQo5H38zTfnBYPBQyaM/5xybWd6nmb4vVk5Dkj39zcK4Ki6Acrfh3roC5EvUV/vgVJQF2B8/cWuBXQeFu7UA50giEmhAjpBEIQxxCvQk71v0uqfqnjInYLiYfy9XbNO1xQPZ0VRbI9XPMT1xgtytC9yegGdy7wV9TnKfb6G8MjInhwA8PkuirS2bq/IQi/0ac1506ZNc3p6SmfU1l7XHQjU+fbt+3ktqdBNA89V6DzHsyKZuLcwAGfU7R0of6dmQVdazFVfuwu6yqIZQDcye27sWEDnZeFOPdAJgrAKqcZMQPK4yTMFd63cBLsIY6y71sB4lXhccZyndbqTC+hc5t3Y2DjcsGrV0X9/9dWl3ZLkWuT1RvaEwzlPyLJw2caNB8rLy3k6pqU954GBgZx8WRaK3e4xzwCzc3IiLBTyDg0Nuf1+vxPd38wCzxjGyOdoO9yPcxmVeAAAIABJREFUMvX8PwLgsLoBQAH0Yvp8KH+3rgWwBHrMdArK35tMYdcCuuFjqQV0ipkIgpgQKqATBEEYg1ZAT/xN1TodgJcx5g2Hw8vGJ3kwVvEwPkCJYazioUtTOSRQPIxOU/FgFE4toGtwmffBg4fLfb5IhLFfCZGIbifp87HIgQMHygHwKKBP+/rT1Od+/2dCLlceKyxcPjow0FBgchW6E65tu83RqQr0ZMhQ+qG3A3gLwCoAV0Eprs8B8BEALiiKjBboyaGeaY5r5wI6DwU6JYMIgrACEy4oUt21fIyxAgCIRqPzo9GoMH4RcZy71njrdECxTtfctQKCIHSNK4gHRFHU4iaz9od2agEd4DjvX9x//zt333138OFt2xZJwaDPU1w8uOajHz34gx/84CCvY8A0nw8uvPDCwR0eT7QpGMxZnJd3rkC3OxDw++rruzgvBCDsj9VimvH7hwFjDAM4qG4A8CMA+9TxFkJplaUtVtYs309DcfpKF96xDC/FNo8COsVMBEFMChXQCYIgjEGORqOVkiTNYozlJlA8+OLfHI1GawCEMFbx0JegL572dYSj4sEoHFlA572Y4b77fv5ue3v7/kTfq66uDvE8lumwadOmOV1d+SVlZbPPDg8f8gFATs7CyL59x7io0J944onZhw9/UPIv//KDAyYt1tuJVO4JZlwUNBXM0AN9usgA3lA3N4DZ0C3fr4OithiBXkxvgdIbcCrYsYDO5QZCCnSCIKyCLMu+aDS6SJbleQlipjHW6bFY7AYof3/iC+C9oiieGu+upS4oHhUEwQ7WrE4toHN93svPz5d+/vOf7wkEAvs6OztzZs+eHfL5fNm4ftI+1+vXr+/586JFrb/cu/eCj4VCo1Veb3Tn8HDeqx5PbOMnPnHEyDjm5ZdfnvnCX/5yQX9LS2leWVlg9YYNzZ/97GdPJxnT6s/0ToXX/cgMPdAzQSuA99X/F0N39VoKYDWUv2sdUGKmZvX9U1nQZVcFuuEtqUiBThBEMqiAThAEYQwsFotVSZJUFad4GBIE4Ux8QTwaja4CwPx+/x84WqebBUcW0MF53n6/X54/f34w+Tu5kPacjxw5ObOwMDQSifzinLLI7Ybsckny3r17S40soA8ODrqfeGLbov5+oeC1117ruOaaa3qNGosAMLmDh9GfG7P22pvqGEYnNcaPEYOuPAcUBWAtdPvCj6o/Mwy9mN4M4OwUxzEKLn3JVbioNhhjpKYgCMISyLLsjUajS6AUxbU4qV39ekQUxRFZlvNisdgNOTk5D7nd7g4bLCSeKk4uoHOfd35+vjRv3rxsxU/TehZxu93sJ/feu+MX9947/ODOnXPFSMTjqanpufbmm5tuvfXWU5k6yPFs3ry56ul///fVS4eHc6/LzY2c7uwseeXo0eqO9va9/3zXXUeMGtdi8L6WeTxHWzlu4uUMNn6MAXXTCupl0GOmFQCuhPIM3wY9bmrD5M/1vBcD28nCnWImgiAmhQroBEEQxiDn5ubuhvKwO6EVUywWuxiA24HFc0B9EGaMOTEZBDgrCTbtoOeXv7xnR19fXyJbTtTW1hqqpH/iiSdqz5zJK5GkKnHz5ufnXX311b2kQj+PTPe9tis8EzVGkmyMKIAP1Q0AcqAU1DWF+hL19UHoSosWAEMJxrFNX/K4sQyfE1m4EwRhFdxud09+fv4T0P9mnEc0Gq2JxWIQBCHqwOK55mDl1IdP553waVJeXh6552c/e39wcPDAwMCAZ/bs2SG3223Ys0ckEhFe2LRp8VUjI76v19b2iOpndE5PT+GjL764sOXTn26pr68fNWp8i2HEeciGit8Oi4J5xkyTzaNX3Xar758JPWa6HMDVUBYrt0KPmzowNm6xqwKdeqATBJF1qIBOEARhDKmqjGUHF5CdrkB3DHGJzrTPdVFRUayoqIh7X8rBwUH3li2757vd10dLShpHm5p+Pvu1114rIxX6ObKhpACM+xzxSghYuV9gOmOEARxXN0BpY1IH3b5wmfp6P8ZavvPqsWc71Yb6bEHJIIIgrICMFGIm9V+nxQ0aTlagO5GMnGte8dOxY8fyYl1dxdcUFw+LcQtcri0pGX6ivb3s3XffLa2vr28z+jgciIDJrxWrK9CN3n+2FOiTwQB0q9u76s9XQleorwFwDZTFyqegx0283bTsZOEuYmp2+QRBOAwqoBMEQRiDZkGcLPh1sprA6QV0p83bkmjq86qqjX1ud5HU17ewkFToxDTgpXTgUXSeTuIkBOADdQMAP5TEUD2U5NDF6usRKH3UF0JJEBllp2o71QYp0AmCsBCpxkyAs+MmJ8YOTpw3Y8xa6wZ8Pp/MRFEekSRX/OsBSRJll0vOzc2l4pT9MIPjVab2b+bF0wzAGXXbAeVv4CzoCvWrAHwESkEdUHqqHwHQNY0xk0EKdIIgHAUV0AmCIIxBRmoPek5MCmg4vZDs1Hmbnu7ubu+jjz469+abbz6lqc89nmIJAEpLPzpEKnRTYNWEAK9EjdkU6MkIAmhSNwDIh5IU2gAgD8Cn1Ne7oCstTkEpxGcC3gV0Hj3QKRlEEIRVSHqvUi3MAec+Pzs5ZnTqvE3N8PCw66mnnprdtH//TAgCCxQWBp9uaytr8PtDxR6PHJZlYVNXV3G0omJg3bp1Pdk+XpNgRJyR7dUWVnfVgsFjZDrGkAG0q9tbAFwAqqAsPl4GYDWAtQBGocRKmuV7Jj+DvBZra2NRD3SCILIKFdAJgiCMIVUFeiqWhbbEwYmwbAe52cRU57q9vT3nwQcfbPjmN7/ZVFxcfE4Z8eSTT9a++GLbwuPHf1ra0eEtkaRmsaXll+XKdxlCoXzvM8+8OJcK6GOY9Lp+/vnnK3bv3lv+wx/+74MpKvcn2p+priGTYoeehAEAhwAsh1Ikfwm60mIBgMvU952Bbvd+CopiPR1sp0BXC+i8klsEQRDTIWULd4e3vnLi3J0aN5nmXAcCAdcjjzwyZ8+ePVUul0tet27dqRtuuOHM3d/97kr5wIG6JYxBAtAvSeIrADo6OkrmA0I7gO6SkuG/+8pXdhcUFFBxSmfCa3rHjh3Fzzz11Ny+kydL/aWlgVXr15/+1Kc+1ZYkdsqWhbtprtFpYJYe6NNBgtIb3QulgP5rAIXQXb2ug6JaH4ESL7VAiZ36pjEmTwU6YHA8I8syxUwEQUwKFdAJgiCMIdXiMFm42yP4mgoMcGQC0HQJsF//+tcXPfPM/hV+/4PRu+666wgAdHZ2erdta7ogEJjja27eX3bNNWVHcnNHYowF0NzcXFxbW3vW7Z7BampqhifZtenmaiBJr+NgMChu2vTXRR0dmLF69bbODRs2mFmFwkuBbiRm7IE+3XGGAOxXNwCYAb1/eiOAVer7OqArLVqh2xkmg+ffYS490MnCnSAIC5FKAZ0s3J0XM0EQBEYxU/YYGBhw3/b5z28camqqXy7LrlHG8ND27Usef/TRtobBwbwfVlaercnNjQLA0UAg5197ewvZkiWn3uzsnOH1emO333773o0bN5r5ud80bN26debvf/rTNQ1nz+av9vkibadPz9x68GDt6VOn9n3nn//5aJIfz+Y1Y2UFuhl7oE93nBj0QvlrADwAaqAvRL5Bfe8w9GJ6C4CBKYzFJZYBp77ukiQJjDFqM0EQxIRQAZ0gCMIYyMI9OU4tJGs4dd6moKWlJffll/cuGh6uzXvuubcab7/99hOlpaXRzZs31/b2lhZecMGdna2t/1pWVVUZ+NrXvnbij3/84+zf/vYP62++uWjHr371q13ZPn4r8fTTT1efPu0pi0Zr3U8/veWC9evX90yjfzxPpbAR2KW4zbOAnkgRcBbAXnUDgFLoSovlANaoP9cGPTnUBiWpNJVxjICXhTsV0AmCsAqp3BOduvBWw4mFZA0nztsUc77//vsXhg8fnvNgfv5IY05OFAC2joz4vn3w4IX5M2f2acVzAFiQnx+uPHas8L82b77KxZgLjAnvvPbaius/9rG3HnrooTen8exve2RZxlOPPbbosrNn879VV9ftEpTT/0JPT+EjL7648MTNN5+eN29eMI1dW12BbqcCutFo44z/exoFcFLdAEWpXgc9bmpUXx+EXkxvhrJ4ebKxeMWBMHossnAnCCIZ9ARDEARhDGThnhxHJsLirOudiGnO9UMPPdQwMFBaMGvWj3u6u3OLf/vb387T1Od+/7qw11si+f3rQtu2NV3Q0dHh/a//+t3lgUBB/vPPb7+ks7PTm+3jtwrBYFB84YW354viKrmi4pMDTU2j1du2bZuZ5MeyeZ3wtqQzCjsV0FMZpw/AHgCbAdwLxb7wJSh2hZcC+AKA76r/rgVQC6Vn4FTHyQS8LNwpGUQQhFVI2cI9hffZFacuurbDM5ll2fPmmxesB5hWPAeAa/PyQkskSTwWCPjj39va15fX2tVVcbkkuY+7XLFWtzt6lySJf3366bX33XffAv5Hb0oSfoZPnz6dO9raWraupGRYK54DwEdKS4fzhof9u3btKp3K/jhh9UXN8Vi9SD+VcSIAjgN4GcCDAH4K4E8AjgKYBeDjAL4F4BsA/gbAYgD5CcbiWUDnYeFOMRNBEBNCCnSCIAhjSCnJo9rSOXUxkyML6KB5Zx1NfZ6T8w8Rr7c+5nLdID/33J8bXS6X3NtbWlhXd3UPAJSXrxtqadle/pOf/OTiY8fO1OXm/iw4PPwv+ffcc88yUqGfR8Lzq6nPKyuvP5uTUx7r7W0sTEGFnqp7hxWxizrcbAX08fSo2y51HxXQlRarAKyDosg4DUVpkZvmOOlAFu4EQRBjkUAW7slwcgHdafM2zTNuLBJx54riecdT4HZLR2RZaA+F3NU+XwwAXmlpmXWKMdzndscq1CLw/3a7pXdiMfdTTz654jvf+U4yG3KncN7v0+v1ykwU2aiihD1HWJYFSRSZ2+12an9mu7TWAsxTQB9PCMAH6gYAfigKdc3y/WL19V7o6nRezl28LNxFsnAnCGIyqIBOEARhDKkq0J2YFNBwaiFZw6nzzjqa+ry8/BP9AFBcfMtQV9fzJf/zP1svBq4TOjqeLdbeG4kUuV988eVLJGmJKzf3b0OSdMz1/POPXPLd73buq6ysjGRvFqZhwutYV59fJefklMcAoKzs+qGmpp9Vb9u2beYkvdAn+2wY/bnhkajhlUAxegwzF9DjYQA61W2nus9Z0Avqa6H0BwSAz0JPDnVmYOxE8LJwJzUFQRBWIakCPc7ByanPz06OGZ04b1PMuWHFilOvnjo16wuSJM50uWQAOBqJuA94PNGiefPaftzZWbxcEFiUMWHzyIhfALByXMF9GYBD/f1F2Th+s8EYgyCcf2qrqqrCpQsXtr+4Y8f8JQUFo4Vutywxhie7umaESkqGrrnmmu40h+Rh4W7luMmOPdCnO04QwBF1AxT1eb26zQVwifr6lVBaaLWoW2ia4yaCl4U7xUwEQUwKFdAJgiCMQSugJ4Ms3J03f6fO2xT97k+dOuV7+eW9iySp0T04+PQ5O7JIZLbn7NmdBVdd1XbI4+k+V9xqb28rOnmyt8bn+7ewIIjIzf1yaHh4E6nQU+C5556rOnXKNTMa7XQ3N/9nufZ6MFiQ88wzW+dOUkAHsqu+MY3yJ03sUqTXxsn0XBiADnV7B4q64VooCSEXFHX6BiiJoFNQiunNUBTtmTgWsnAnCIIYS6oxkymeJbOEUwvoTpy3aZ5D77zzzkNf2bnzgi+eOVO+3uWKhRgTXmFMyL/oouYHHnpo60svvVR1YO/eClEUZamgYLTtvfcWhaA0WQaAKGN4GWCz6uo6szkPK3DbHXcc/mVHR9G3W1tnNjAmtANonzEjcONtt71fVlYWTbqD7GGa6zUNqICenACAQ+oGAIUAvgngLIAGAJepr5+Bvgj5NIBwBsbmZeEuMsYoZiIIYkKogE4QBGEMqfbpY3C2FSFASRGnYIp5nz171lNSkjvk9x8LAcfOvV5WhsG8vNnh73//f+2tr68fBQBZlrFu3bV/KwgXCS5XnRyLHXQBAGNrheeff4VU6EmYPXv2yIYNpUeAASibRi5mzZoVSHO3RichrK5w13C6hftUkAEMAYgBeBxKEX02dIX6BvW1IHSVRTMUK8N04GLhTgV0giAsRAxk4Z4MJxaSAZPED05l/vz5wd88/vgzDzzwwOK/7NlT73K7pWWrV3/4rW9963BZWVn09ttv1xYZYs+ePUU3f/Sj8z4eDvv+H0GQfYKA/5Ik8ajbLf3qjjvezfJUzMKEn+HGxsbhf/3Nb7b/5S9/qWltbS0oLioKf+a669qWLl06nMJ+J/qc8FCgG4kd4iarF9DHM6SOtQ/A+wBmQI+ZFgO4Avpi5RboBfV0FoHwsnAXoDyHEARBJIQK6ARBEMZAFu5JcLAVo1PnDZhgzkuXLh3esuXPz6Xy3tOnT+e2tJyucrs9sUjkJk1MAZcLUjgseZ9//vlqNXGUcSKRiPDMM89UfexjH+vwer1WSB6ed4yrV68eWL169UCiN5sco3/fdilu26WAro2jLXyToCjPTwF4HUq8VAO9F+BGKAmdAPTEUAuA/jTGMgy1gO7UnpkEQViLVGMmpPA+u+LUmNGp8zbNnOfPnx+8//77dwGY1HlrxYoVg/c98MCmn/zoRzd85syZCjCGGaWl/d/75jdfvuWWW9qTDJP2fPfv31+wZcuW2aIosptuuqm1oaFhJN19cWLC59rKysrIHXfc8eEU9mWG68TKcRPP359dCujjxzkLpZi+T/26BHrMtAzAaijxSDv0mKkVqRWsucyJFOgEQSSDCugEQRDGICNFO0KHWxEC5gj8soHT5m2FIvAY6uvrR5955slH+vr6vOO/5/F45DVr1qRaMJsyDz744AUPPfTnNV1dXW997WtfO2HUOBkg49cxYyldKlZVU/BSa/AYw24F9InGiUG3cQcUZ1KtoD4HwCL154egF9SbAQymMVbGIAU6QRAWIum9ShCEVN297IpTC8mA8+ZtuZhJ42//9m87br755t++++67xeFwWFyzZk2/2+02ZD6yLOOf/umfLn3r2Wcv9YdCOQzA/zz44OiNt9769g9/+MODRoxJjMEOMQ1ZuGd+nH5126N+PRO6Qv1SAGuh/M1vhR43tSPxcwCvAjr1QCcIYlKogE4QBGEMU1GgO9mK0Im9DC2bFHEiy5YtG+I9ZjAYFDdvfnFpT09pyebNW5bedtttJ/1+v2OUpIIw6S2Bx/3C6CSKHRToZihsZ2ucCIAP1Q0AcgDUQU8OLVFfPwtdadEMQLPgJAt3giCIsaRyTyQLd+cVkgGKmyyHKIq44oorDHegevjhh+e889RTV3wLkD8/Y8ZgjDHhwUAg73ePPLJ22bJlPTfddBP1XefzHG3luIkK6OmPk2puokfd3lN/thy6Qv0KAFdDWax8Gnrc1KHuX/t7b3QPdEGWZbJwJwhiQqiAThAEYQypqqudmgwBHKpAjysOOmreKk6bc1rzffjhh+e2tbHysrK7BtraflT+8MMPzzW5Cj2jZHlRjdUL9AAV0HmPEwZwTN0AIBdKQV1LDi1XX++DkhQqAofrjAroBEFYiKkU0J32LKnh1JjRqfN24pynxF+fffaiy6NR1+0lJQEAgCCwbxYUBN4eGJjx582bF5i0gG6n88prLlYvbtu1gJ7OOAxAl7rtVPdVCd3V60oA66EsVj4N4Mx0DzYVJEmimIkgiEmhAjpBEIQxpGzhDnsFUlPB6Ykwp807qwqSSCQi3H77V9auXXvFaaP6lmcCTX0OfAR5eVeFgsGP+CyiQs/k+dUcPBJhdHKAl9LBaKiAPjUyqQofBXBU3QAgD0ohvR5KcqhUff2r0JUWLerPZRJKBhEEYRVSiZscbeEuCIJTC8nZXliZDbIWMw0ODrrvu+++RXu2b78wPDKSWzlvXsdnv/jFgzfccEOXgcOmNd+R/v78uS7XmNhIFARcIAhsb39/XmYOzRB4nl+rK9B5YYfYj1cBXVOFZ2IcBqVIfgbAO+q+q6DHTFeo7/sUxrp6dWdofACkQCcIIjlUQCcIgjAGsnBPjpML6HYINC3Fn/70p5q9e4cuaG9/seSTn/xka3FxsSmDJE19XlT0+QAAFBV9PtDW9oqZVehO/PxOB17qcKOxWwHdyHFGABxWNwD4OBSFehuA+QBWqq93Qe+ffgqKsj1tSIFOEISFSDVm0u5tTsSpBXTHxUyqW9n/z955h0dRrv3/+8xsz27aJgFChyCdUKQoIFVsgIrlAO85AmLBYy8cRY76+jvHV8RzBAuC9YgNEQSPShNUEJHeEQiBJAQS0tv2MvP8/thdsoSUJdmZZHeez3XNFdjszDPP7uxm7vt7f+9H9vdaFEU8OmfOdba9e3vcwfNCikol/LpvX/dFGRntBEHYOGnSpBbl6E5NSyvckZ3dzk0pNP4ObxZRJLsBXJWWVty8Zyc7zfE5iab1yaUkEpzhLWUcEb746DyA3wAkAXgYwHEA8QDGA+DhKzrOQbWo3qTPO3OgMxiMhmACOoPBYEhDSOKwkt0EYAK6EufdLHN2u91k9eoNfTyedFVBwQnz8uXLuzzxxBOnGt5TXgLuc6+3t0oQyjiHo0wDAF5vb1WEuNDDRXO3cI/0ZC1r4d64ceT6bAkA7AC+8/8/DtVOi14AhqHakZEDX3IoF752hiHDBHQGgxFB1Nd5BsDFmAlQ5v0zoNzYQYnzbpb70HXr1rUuOngw7TWj0XaNweACgDtiY61PFxaav/j44/6TJk3a2BznVRez77vv6NO7dnWfU14e/ye93uGllHzudOoqEhMr7rvvvuPNfX4y0RI+G5EcN7EW7i13nOAxDsFXXKwC0B7VcdON8JmRbLjUoV52JYOIokjAYiYGg1EPTEBnMBgMaWAt3BtA6Ykw1o5QHqxWK7dmzZp22dne1snJ95ZXVm4xfv/9f3vPmDEjq6W50I8dOxZbWWmNMRgOOd3uRzWBxw0GOCsr3cYjR47EDhs2rKI5z1Em6rtW5AzapSBaxG25vr/C2Vq9PuQsnKgp1lcCOOzfACAB1eun9wNwrf/c8lCdHDoHwFPXAKIoBsZhySAGgxEJhFLApOgW7lB21zKlvueysm3btlSTw6EZFBdXFXiMJwTjDQbHK2fOtLFarbzRaGwx9xUjRowom79w4XfvLlo0/KmzZ1tTQtC6T5/T/3z22d/79Oljbe7zqwO5r2Up72+jyYEe6WMEjyM1csbiNcfyorpb1y8A1AA6oDpu6u3fxxL0vBwA9eZPBEEAWMzEYDDqgQnoDAaDIQ2shXvDKFlAj1Txr6nI+l4fOnTINHHiHbNjY41WQmZRnS7Nw/OxVQUFm1qkC33IkCEV33zzn1V2u52v+TuDwSB06dIl3Gskh42ggpiwHC6Mx2rM2FInUeSs2JeKaHSgyymg1zdWuX874P9/EqqdFoMAjIQvyXMe1U6L8whK/Hi93sBniCWDGAxGJBDK92/gOUqOm1jMpBxke683btyYPPeppyZb8vNTiShyPauqYh9NTi57IimpAgDKBIHndDqHRqNpcV2wJk+eXDBx4sRvMjMzY9RqtdiSY6Ugou2allqgj/S4SU4BXU5RW47vg4bG8gA4498AQItqQb0zfIXIgE9Az0G1oF4VfBBBEFjRMYPBqBcmoDMYDIY0hOqSUGoyBGACuhLnLSvPP//8KIultclqvWDs2fOmUwCgVqcIPH+D0FJd6BGS+AlGquu4oQRAtCWfwkm0uNyjcRzgyl31Jf5tn///KagW1IcCGAWfI+McgJxt27ZV9u7d+6T/uS0u2c1gMBi1EMp3lZLjBgCgCuxeBSgzZpLtHjcnJ0f38MyZM9LsdsMsQmgsQL8TBPXCgoIUAyHCaKPR+a3Lpek5fvwRjUbTIu+9OY5D9+7dbc19HgokGj6X0bYGenO4wqUkUDAX6lguAJn+DQD0ADqiOm7q73+8DED2qVOnCjmOy/EL6CxmYjAYdcIEdAaDwZCGUB3oim3hDv+NsEKTQYDy3ndZky6HDh0y7duX3Qu4k4jiZu7s2QfaxcdfVwUAoujgior4hJUrV3acM2fOmYaO1QhaZIIpSpCjXWCkO9Dl+m6JJmFbrlbxQNPXWy/yb3v8x2oFf+vCioqK4dOnT9fo9Xq0a9cO58+ff4oQ8hOAg5TSFlUsxGAwGEE0+P0b1OmGOdCVh1LnLTnPPvvscKPDYfiU571dOY4KlHITvV5+OkBeKCxM6Ulpqb5nz7PPzZt3qLnPlVEnDS19JfX9bSQ70KPF5Q7IGzNBprGa+v44AJz0bwBgQLWY3umDDz4Y9MUXX6Bz586U53lCCMkAsJVSWtqUk2YwGNEHE9AZDAZDGgICeijPU3IiCFBmUkSpAqts7/Xzzz8/yu1uo1Gp/ub2enVqj+djzY032g7qdDoR0ABIQ3p6eplc58NgBBENCZRoc4a3pBbuVwIFUODfdhoMBv7jjz/uu3Pnzj5ff/11VwAvAHgVQBUhZD2ldFqYxmUwGIxwEup3opJFZEXOPcxLBEUSsrzXmZmZbfsC6MpxFAB4QkS9SkWvFwTVdoDc/fzzG+6+++5zBoOBuTPDQzR9hqOhYDeaRHq5ioGbYw30cH3/2AEc92944IEHUrp16zZo48aNvbOzs7sBWA0AhJAjAB6hlG4P07gMBiPCYQI6g8FgSANr4d4wShfQlTZv2RJgAfc5IY9RtbqNwPMPiG73Os2FCxfiVqxY8XM4x8rPz9c+88yz1/71rw8eHjFiRMQL8lu3bjUDwOjRo6+08lquNdClDtrlcKDLARPQr3wcuZLDkiW4NBqNcMMNNxzq27fviffff/9vABIADAYwBkBMOMYghIwAMA/ANQB08K2/vpxS+ko4js9gMBRJqN+/Sm1jDigzdgCUOW/ZYiaj0ejIA+CiFFrie5kJIfQcQFVarWvmzJmBOhgDAAAgAElEQVRnwzFOXl6edvHixX2O7tzZlQoC123gwJxHH3vsaKS2Xvd6vVi8eHGPjIwMc//+/QsefPDBMypVyOn1sL2/IXwfRoMDXQ6iQUBnLdyvkG7duhV169Ztw7p16wDgK/gE9DEAxgIoDscYLG5iMKIDpboeGQwGQ2pYC/eGUbKADih33pITcJ+r1TPdAMBxnSkwCVu3HuxfWFioDudY77//fvd9+2xXLVv2cXo4j9scWK1W7t57H/vTrFmPTrNarSHdI0qUyI5mt1G0uMOjUUCPlBbuDeLxeDhCiEgpdVNKd1BK/0kpndfU4xJCpgPYCqAcwF8A3ARgAdjfMwaD0TRC/f4VodwclhKF5ABKnLcsc549e/busxwnviwIqipKiUApvhUE/huAdE1PPxWOMUpLS9UPzphx85lVq669/cKFpKlFRQklP/ww5JGZMyedPn3aEI4x5GTz5s3JfdPSHvn0lVfuPLRq1dilf//71P49e845cOBAbHOfm8zIsawWEPmFzUxAb+FjiaJIAAiU0nOU0k8ppTMppScb3LEBWNzEYEQPSg0+GAwGQ2pEhL6enyK/i4Na8inxBlKpSTDJ5+z1enHgwKmelIK43dO0Tud4rdM5Xgtsh9Op17/44ouDwzVWfn6+duPGfb1FsQ9/5Ehxp82bNyeH69jNwfz586+uqDDFV1bGxT733HNDmvFUar1Ojhw5EvvZZ5+19f83Uh3okOH4ciWDoknYjtQW7rXi9XrDLtITQlIBvAdgKaX0z5TSdZTSbZTSjyml/wznWAwGQ3GE7ECHMu+fAeXOXanzloUZM2bkDrnhhp0fcJw4RBBUgwRB/TBAvG3aXFixYsW6cIzx4YcfpgmnT7dbFhdX9WRCQtXDCQmWjxMTK+Ly81OWLVvWKxxjyIXX68XjDz54d8/y8oTlHCfsV6tdHxAiti8qSr73L3+5q7nPT6gQVO6zbn3wY1SkcP7hNEk4bCTHTdEi0gfGicS26qGMJem8BEEgAIRwHpPFTQxGdKFI0YbBYDBkIpSAX8lJAaUL6EpDljmrVCrccMPwPSNG6A8MH+45WL2RgyNGtDt45513ZoZrrPfff797eXl8bJs2L5Q4nT20y5ev6BOuY0tFTk6O/r777h+RlZV1SXLFarVy33//y9WUjiOUjiPr1m27OlQXukRcdr0sXLh46LJla/qePNnkgvDmJFrE7WgTtuUU0CVfo9DvQA9rIgjA/QAMABaG+bgMBoPB1kBvGKXOncVMEvPVV1/99OHq1cvajRmzxzB06OFZzz23+uCxYx+YzWZvOI5/5ODB1CGUkk5q9cX7kiSeF0fzvJBx8GD7cIwRboqKijR///vf+980btztt0yYcOs//vGPPhaLhX/vvfe68uXl8c/zvHckzwsGQnC9SiU8zvOC68KF1uvXr09p4NCSfobL3inrXfzP4qHuHJ+ITikllV9UJpa8VjLMW+jVhHk4uRzo0TJGNMVmkGmsQC5CUrE+4EAP82FZ3MRgRBFsDXQGg8GQAJPJRC0WS6gt3EEpBSGKy4koWUAHlDtvyVm+fPk2qccIuM81mns8PB8nxsZOsx458lKnzZs3J48fP/7iWugrV65st2nTT53fffetHTqdTq41luvkzTff7PPzz1npRuNb7sWLF+8JPB5wn2s0s1wAUFn5U+xzzz035J133tkV4qElDaI3b96ccvx4SRe3u41+1aq1eOGFJnejrgs51kCPhgRKtAnbkovaQUiRpLkEKRzoAEYCKAPQkxDyHYA+/v+vAfA3SqklzOMxGAzlwFq4N4xSBXRAmfOWdc7jxo0rHTdu3EYpjq3War213SBYRJFTaTQeADh69KjqvbffHlN05kxrnclkHzxuXOZjjz120mAwyB47FRQUaKbfccftruzsdmMohQfAumPH0n7fti2tS/fuBVpKub4q1SXFBekcJ2q9XtWRI0fMN998c1EDQ4T7fvPi8eLvjT9Z/I/iocWvFA9Nnpe8x3HAoXXsd5hi74jdqmqlcod53MvGDzNyucPl6gwWLbGZpOuS10DWFu5hPiyLmxiMKEKpwQeDwWDIQSg3YYGbQSV+HytZQFdqEixq5hxwn8fH324FgJiYa501Xegul4t8/vl/+x04YOu6YsWKDs13tj5Onz5t2LLlQF+Xq4t2y5aDfTIzMw3Ape5ztbqvoFb3Fa7AhS7Le7p8+Yp0t7u3JjHxMfv27SexdevWRDnGlQg5Eg7R0CY+ME40CfWyjCUIAiGEhDvp3BZADICvAawAMA4+V8U9AMLS5pXBYCgW1sK9YZQ6dyXOO6pc96PHjMk5pFZ7N9lsF7tf7XM6NdsIoUNHjcrauXOn4Z0XXzS127u3+yMOh/bmvLyUA598MuLpxx+/RhTlrz1+4403+nqystqtMhjsi+LiLO/ExVmW63SOsuPHuzgcDpWd48RfBYEP3mebIPAOjhNHjhxZIPPpXvLZUKeqXckvJO8mKiIWzi+8zrnfqY67K6449tbYfKnHlpBIbhEPRJ+AHnVroPtbuLO4icFg1IkSBRsGg8GQi1BbuCOE50UjbO7KImrmXFpaqt60aW8vl8ugKSx8PSE//+/mCxdeNHu9btXRo0Wdfvvtt0QA2LBhQ9K5c6oUrzedX7NmSy+n09ms911LlizpXVWVZExOXlRisSQZlyxZ0geodp+rVCO9Xu8p3us9xatUo7zNsRY6pZRQSrUI+l4IuM9jYu6xxcSMcNtsXfDZZyt6S3QKcjjQpSZa2sTLPY5cWVpZWrgj/E4KDoAWwCuU0oWU0l8ppf8GMA/AcELImDCPx2AwlANr4d4wSp171MQPSuXPf/5zbtdRo4686PGo7ikuTpxdXJz4qMOhjx8w4NTDDz+c8c0XXySnl5dzr7dpU3JHQoL10ZSU8mdiYhznd+++avv27Wa5z/fgzp1dricE3VSqi/dRAzUa7zWUcqVFRbHaNm0uLBAEfrnHoz4iCPx7Ho96iShyiWlp2cOHDy+X81wppWrU6Cyraq1y8WbeFvi/ppvGKfVpSHTcaIqZACagN4ZIbuHO4iYGI4pgLdwZDAZDOgSE2MI9hOdFIxTwCWbNfSLNgFKTYFExZ41GI44c2TvTZrPlAHmX/E6l6iqmpKQ4PR4Pvv32p1RKr6OtWt1Wlpv7ZMqKFSs6zJo1K6c5zjngPtdo/upWqVJEjeZP7i1blvTJzMw8tnPnrjRCQEXxBU4Uq4srCaF0167dXQGE2sa9TiilvCiKRlEUTZTSiz8ppSYARkqpiVJqhK9SmwDwADgN4PSXX65Odbt7a2Jjh1sIITFm81QcPvy3Dlu2bEkZP358Q20SWxrR0l49GgX0qHGge71eKRzopQDSAPxY4/ENABYDGAjglzCPyWAwlEGo31eKbeFOCKEsZlIUUTNnlUpF3/vgg99Wr16d/csvv3RwiyL352HD8qZNm5ZbUVGhKj59Wj8rMVHkXK6L+1xnNDriL1ww7927N3nUqFGlzXj6l7Hqv/9dMe2uu+56/uzZdjpR5J0cJ5i7dz+zZu3aNSHs3uD7SikFpVQviqJJFMWLMVLNnwBMANT+3foAOE1Ferr036Wx7ix3snGC8ZRQIvQtfbu0TcLsBL2mk8bRhGk3ai5hOj5r4R76ONEmoMsylv9va7gFdBY3MRhRBBPQGQwGQzpCSQYFRGROaWugE0KU7EAHlDfvqHGQmEwmYeHChfvr+j2llN+0aROys8XYpKQp+VptWy8hI7FmzZZe06ZNyw1lLfSsrCz9q68uGDhv3nMHunTp0uSER8B9bjbfXe6bw1220tKVCUuWLOmzfPkn/926dWvr2vYbPXp0va0IKaUqABAEobUgCEk1Ezzwi+MA9DV2FQFYCSFWQoiF47g8QoiFEGL1er2t/cc1nzhxYmJenpPExXUVKP1Oa7NB0Os52Gxmw2efreg3fvz4LU19bWogp5AqFXLNIZqEbbkFdEmdFF6vVwoH+mEAQ+v5faR/bhgMRvPBHOgNo+S5K23eUff3lOM43H333Xl33333JZXHMTExAlGraYXHc8l7bBNF4iIEer3ei3rIz8/XLlmypOeR3bs7EZ4Xrx4+PPvxxx8/ERcXV+9+9THgmmuyNp8+3eE+r5cPuNAPuN2qnYSIt48YkZOWlubYe/Dgpzt27Eg4cOCAefjw4UUDBw6saui4/i5bPACd2+3uVo8wbgTA19jdSQixArAQQio5jjtPCLFSStWCIJjh+4wMtG60jhAqBcH8uDnXMNRQJFQK7tLFpWLxK8VDW/2z1e8SrYMeydernPf/0Sagy9G5S84W7ixuYjAYdcIEdAaDwZAO1sK9ftjcGc3GO++8k1ZaWqp/6aWXjob72Ha7nfvhh19AyHBotW29AGA2T6nMzd0esgv9nXeW9N66Na+fybTE9cYb/97XlPPJycnRb9lyoK/X211VUbEkNvC412tWbdlysM9jj2mPPvTQQ2cCj/udDzq/SzzF5XLV53zQAIDH47ktcFj4hHGLXxwvCYjkhBArx3EWjuMshBBHUBHNJYiiOFAQhDIAOQcPHryKUuu42NifDVrtb3qO43gA6NrVreZ5YwcAiQDKmvL6yEy0iNvRJmwzB3rDfAPgQQA3wZcUCjARvvnsDvN4DAZDOYQsoCvUhQ0oV0BnMVMz8tNPPyUtXbp0YP758+bWqallc+bMOTBhwoTicB3fZDIJvYYPr/hh3bqYdLVa1VGr9bpFER+UlMRb4+IskydPPl/Xvvn5+dqZf/rTZOTktB0LX+uqX06c6Lzv9987f7Fq1TqTydQoQeyZZ545OnXHjrS7srPbjaaUeAG6jRAk9up15pFHHskIPG/48OHlw4cPL6eU8oIgxIXaZYtSCrfbfZX/MLYgYbyY47isQLwUFDdZCSG1FgR4PJ62giC44LsHI+q26qtMN5nSDEMNrQBM5ON4Yn7SrLVvs9v4ZL4dgFz4YrVwEA0OdDmOzxzoTR9L6hbuUhQes7iJwYgimIDOYDAY0iGigcAiKMGt5ISIUueuxHm3iDnn5+drFy16f4LL5dbeeuutZ/v371+ra2D//v1x+/btMz/44INZV3L8tWvXtj1zxga3O5s/f35+UuBxp9Og+e67n7oHBPSioiJNfHy8R6PRXBKAZmZmGn755Uhvl6un+pdfjvTOzMw83q1bN3sjpgrAJ6Z17doqPzW1uITnt6pUKpWa53k1z/Nqk6k7IYRcb7fbdTWE8ZrOB1dQgsfCcVx+kPNhtEajWaFSqXIJIc5wdtOYPn36qenTp58KemgogBsBFAFoB+BRAOXwt3sHkA1fDq2xyLEGejQkUORaM1yuZJDk65LLOZYUDnRK6RZCyA8AXiSE8PAt7TAYwIsAvqeU7gjneAwGQ1GwFu4No9TYQanzbvY5L1myJO2fL710VytB4IdSSvZkZNB7tm0b+NwLL3zzxBNPXLw3Lyoq0qxYsaJTVlZWfHx8vHPy5MlnBw0aVBnqOLNmzy7+14ULKQ9u327qKopcEYAik8n2p7/+dUf79u2dAHDs2DFjfn6+fuDAgRVJSUkeAHjrrbd6k5yctp+bTNZOarUAAEfdbtWs48c7L1u2rNvcuXNPNmbeKSkp7q/Wrl379ttv9z28d2+aWqPhp44dWzZjxoxKnufH2O32i/GS/2fIXbY8Hs9QQki5Tqf7geM4W5iKHQP3lFSXrssAEBD5dQAe5mN5m2mSqR2AnvCJ59mojpvCUYQs9RroUsdlUhNtAnrgb3DUjCVFC3cWNzEY0QUT0BkMBkM6GhTQUX0zqMRkkJIFdEB5824xDpKFCxemWywqE6VxeO21hVevWPHlzzWfI4oiFi9eNuD06arWffr0KR8+fHh5qMfv2bNn5V13uVFWVpZdWVmZW/2bFCQnJ9sBwGq18rfdNv327t07XvjPfz76NXj/pUuX9aqqamVs3fq1ktLS+xOWLl3Wqz4XOqWUC15fnFJ6cc08AMY2bdqYPv/840BLwJrXnZ0QoqWUWgghpRzH5dThfKhVlPZ4PO0EQRjNcVwFx3HOUF+jBqjvsxFIyK2ELwnUCb71xbrBF5QK8LkrMuFLDIXNJRNByJUMiiZneLS1cJfCgQ4AdwF4CcD98CWA8gH8G8D/k2AsBoOhHEIVSZUqpgLKnrvS5t3sMZPdbufeWLBg4gSvV/UFz3t1hMBFKWYIAv/m669PnD179psmk0k4duyY8aUnnxynPX++VQ9RRB5AXlq3rt8dDz/8WygdtwAgOTlZ+N8FCyxr167dfOrUqcSrYmNdc2+99WyfPn2sGRkZMc88/vjYnD/+6MqJIkf0eufoiRP3/etf/9p7dM+ejuMIoQHxHAD6ajTeaywWbu+uXR0AXCagB60vfkncVPOnwWAwPvvss5oau3sFQQh02LL4u2xZgoRya0NdtrxebzohxMHzvOUK3o76IKj78+GEr6j4NIAtAFLgi5nSANwAnzM2uAg5B8CVtHiPhs8lWwO9ceMgmsaSqIU7wOImBiNqYAI6g8FgSEfIa6AjOgKQK0XRc6e02XMjiiQ/P1/7/fc/X03IDMrzrcXt2/+RfujQoX01Xejr169vdeKEvZ3d3kb/5Zdfdx8+fPiuUMcYOHBgVffu3aFWq3O1Wu3h2p6zbNmybnl5QquSkpPxe/bsOTJkyJAKoNp9rtU+7NJozNRguEv47bf3BmVlZQmpqamami3U/T8NNQ5PEeR8IIRcCBLCg4VxGyEkXMFiOC/oUI5F4UvynPJvgK+deyAxNBbABABVuNSd3pDIzxzoLWeMwDhyOd3lGCcwVsQ50AGAUuoC8Lx/YzAYjHBBEfrffiXGDYBy567UgKlZ3+tvv/22rdVqNc7jOEHn7yylJQTzOE783m43rF69uv2sWbNy3vzXvwZ2OHu29f+1bl1qVqlEL6VYUlSU8M377w+74YYbLqSmprpCGU+r1WL27Nk58Im4AHzFgA/MmDE5Jien3b/VandXtdr9o92ue3/lypF/1+m8IARCUDxNOY4Dx/Eiz/Nx8fEpTqfzuprCOGpfX9xFCLHAFztV+btsWWoWFBNCXGHqstVc13SRf/sdviW4OqH2IuRA3FTUwPGkFjflcqDLJaBLjZwxExBFLdwppSxuYjAY9cIEdAaDwZCOUBzoInCxbZDSULSADmXOu9nnHHCf6/UPOggxUbv9fUNNF7ooili58r893e50VWLiuMr9+1/vvGPHjowrcaHXBaUUVqvVsHbt5mEq1Q1qr/eYdsWKlRP79u1bRCk1fffd920o7RTfvfskolJ5uJSUicjJWY/16zdMnjlzhgBfC/WAEH62DueDvS7nQ4RQ33VS3+/KAOzxbyoAHVEtqA+E73N3DtWJoQI0TxKLrYGu3HEAGVq4i6IolZOCwWAwpCCk5Li/s4YSu3YByo0dlDjvZr+H93g8HCiFroZgrCMEEEXi8Xi4goICzYVjxzo8azLZzSqVCAAqQjAzKalyc1FRwvr161Pvu+++7MaewxdffNGhMjc39UODwZWu1VJwnKqvyQRnZaXm259+GnfnPfdU/bhkiSnDaDR1MRh4ADhss+Go0YgHJk7s6vV62wQtP1Wzy1Ygbqqzy1YEUd/1Utf9bX1FyGMAXA/AguqYKQsNFyFLBVsDPfRxosYV7keWFu4sbmIwGA3BBHQGg8GQDtbCvR6CKriVlhQJoLR5N3syKNh9znGtKADw/ANCTRd6wH2emHi7xWDo4zp3rltKQy50SimhlBr8LQHjAEAQhKscDkdqTefDxo0bVVVVGnTseC9stiPYufONLllZWYlGo9H6/fc7dDZbe++ZM+97RJGKokgFu12t+vjj78pvuunGr7p27eqQ5cUKHamu46ZeL14AZ/zbJgBxqE4MjYDPoW7z//60/6cd8jjQpSaaHOiQaRzWwp3BYDCaD9bCvWGoQguuAeW+583GxIkT81987jnn2w6HfikhXo4QiJTiLUHgNVqtc8qUKeesVquKiCIXw/OXCE8GjhNVlBKXy1XT6X0RSqk60EZdFMVESqne4XCMC+6y5XQ645O1Wk3f5ORLbpqGi6K4orKSHz16tH339u2Vsw8ciBkmCB4XpeJuQoSUAQPOTJgwYX1MTEykC+NyUrMIuQOq3ekDUHsRslwOdClha6A3bhwgitzuoihykK8TGYPBiECYgM5gMBjSwVq4N4xSE2HNLiY3E836Xr/++uv9qqrEWEKqqMPxmg4AKLUTt1ujWbjw9au//PKLn4Pd5zEx/VyEAImJtzmPHl3cc+/eve5+/fqR2tbLg68l4CWFMKIophFCKuFzPpRzHJfrcrms7777ST+7fYzRYulU6fW2FUpKPopfuPBfh1966YU9bduarklMLFbXXLpbr2/j4fk681BKorGBdCWA/f6NA9Ae1YJ6P/9z8gFo4RPWpQqko6VVYDQlnAJjRU0Ld0EQJGlFyGAwGBIR6vevUuMGQLlzZzFTM2A2mz0zH3jgx6VvvTXpD1HkR1CK3wnBPkLo7Hvv3ZKUlORJTEz0GNu3L/4xI6P9NUajgyME4Dh+U1VVrCshAaNGjUp0Op3X1rH8lLbmmIIg9AnusmUymfLLPZ6euRUVzs4cJ0AUBYgiPWq36906nbNTp05fvv3ee2TZsmVX7du5syPH8+KkkSNz5syZk2kwGFqqGCb3+9qYe04vfI7zLAA/ovYiZDuAC/7n68JyppcTbS3co01Ajxqx3l+cxuImBoNRJ0xAZzAYDOkQ0ECQFOQQU2JCBFB2MkiJ825WNBqN0Lp1TBGwEQAhHMfxPM/xHBfPt27dppXD4Ri7b9++5MJCvkNSUopKpVrfjuPAJySIKCpqje3bd3To2bMnBWALSvAUchx3OnidPEKIzel0PqZWq9fXXAN9yZIlPbKy7DqT6Z4Kj4cKgAo63T3OnTv/2aOkpOTwF1988nOtJ9/yCWcQLfVnQwRw1r/9BF/xQ1f4EkO9AJgBzIUvcRRwWljCOH40JFCiKeEE+AomvDKMExhL6jXQ5SwIYDAYjKYS6ncia+GuMPxLEilt3i2iaODll18+0qlTp6pPPvpoyFeFhSmt27WrWnTffVl33nkndTqdIymlxjlPPZX84auvJswvL08ZaDJx+S4X2W0wYMLUqejUqVMbr9frxqXLTxXUXF/c7XYPEUWxg9FoXBo8/qRJk/jFr75qfrqoyPyyXu9NU6noBpdL85Eg8FePGXPEZDIJADBv3rzjAI43x2vUSCIpZgLqLkLu5f/9ffAVIQdipjyE9x6UCeihjxM1bdXlHIu1cGcwGA3BBHQGg8GQjlACfsW2cPfD2hEqB0qptHEWpRSUUq0oikZKqSnwM+B8ePHFF40vvvii3u98qFktHy8IQiu32+1OTdW4PZ59dkHY7REE0eP1Cp6kJMF7+rSnMCYmZmtD64tTSuv8PG/a9HN3SkGqqv5kCn6c53nhq6++6jZkyJC9jX8FFEU4LyYrgMP+TQtAAyAbvuTQZP9zilCdGMpF44Ns1sL9ysaADOMExpLTgS7pWMyBzmAwIgzWwr1hlDx3pc5bUiilnL+N+mWdtQI/77rrLtNdd90VA1+uwgygs8fjAQAHIcRy9dVXW3Uvv5zz/dq1+lUnTqh1bdvarhs7Nmvq1KmnNBqNJZT1xT0ej0DI5W+xyWQSFrz55nfzn3rqlilFRUmc08l51GpP92HDji5atGhnuF8PRkgEFyGfATADwGYAbQAMBnAdfGulh6MIOZpiJsg0TrQ60OVo4c7iJgaDUSdMQGcwGAzpCLmFu4JFZKUmg5Q670bN2S+MB9YXrzXBE1hfHIC6xu4eXOp8KAo4HwghFo7jAg4IByEE48aNw7hx45o6zzr529+e/D0jI+OP2n43fvz4C7U93sIhEhRG1HdAqT83BIADwDb/pgfQBb41ANMBXAvfNRWcGKq4wuPLkXCIhjHkTtDIuQa65A50QghLBDEYjEhBRGjfi0ouvFVq7KDUeTd6zpRSlX9t8VoLigEEfhpq7gpfl61AnFTAcdzpoA5bgbjJGnyPMXToUAwdOrSxp1svY8eOLdm2Z8+nq1evbldUVKQfPHhw8fDhw8slGUwGRFFEbcUCTaShuEnqNcpPAPjd//9UVLd7b2oRslxxgFz3/0xAb6FjsRbuDAajIZiAzmAwGNLRYAt3VIvsSkwMAMpNirSI1nwyc9mc/c6HGEppKMJ4TVe3kxBiAWAlhFRwHHe+ZktAjuMsHMe55ZhcqFx//fXF119/fXHDz4wM1q5d22r//sOYP/9ZLjY2NlyHbe7vhOBr1QHgD/9GALRCdWLoJviuy1JUJ4bOwiewNyfMsdG4seT6Xpa8hTtzoDMYjAiDojomqO/7UbEt3BXayhxgMZPvgQa6bAWtL27C5euLC/DFS4Fi4nP+nzXjJltDXbaaA41GQ6dPn36uuc+jKaxatart15991s9bVZWsMRhMaQMG9H/uueeOBtrQN5Hm/l6gQT/z/FtwEXIagH6oLkLORnXc1NzFEHLFTAAT0BuD5DETcLF7IIubGAxGnTABncFgMKRDBGvh3hCKTAb5K8+jdt6UUr6m8wGAThCEq+x2e2qQMG7A5a9DsPOhiOO4rIDjoYbzQa71ihl1YLVa+c8//65bRYUeK1eubHv//fcXhfHwdQXLUgft9X0uKYAC//YbfAnKzvAlhnoAGApf8J2D6sRQSS3Hj4YEChPQW/BY/jXQWSKIwWBECgEHekPfj4qMG/woee6glErh3G12auuyJQhCFwC83W6/K5xdthjNw4oVK9p9+MorN1xjt+vGtmlD8goKNGu/+mr4X7OyEpd/8cXPHCdpGkgOB3pdBBchA5cWId8IX/6rDL54KROXFyHLEQdEUzwjt4Aux9JXssyJtXBnMBgNwQR0BoPBkI4rEdCVGtUqNRkUkfMWRVFTj/MhuCVgzfXFA5+FGACVfquMw8wAACAASURBVLe4tQ7ng1zrEDOayKefftopN5fGG4198e23P3ebNm3aEaPRGA3BZ6iBugvASf8GAEmoTgyNB3ADgEpUi+nZiC5xO1oSToGx5BTQ2RroDAaDUU2o8VBE3j+HCaXOPfjaaHHu6LpoYpctD3xLJOkjqcsW43JEUcTXy5cPGGW36/4vJaVMNJuNxOt19/Z67c8dPJi2adOmYzfddFNTC5Cb+3MR6viF/m0HLi1C7g5gCC4vQg52tktFNMUz0ehAl2VOzIHOYDAaggnoDAaDIR0NJsiDxEIlJkQAlgxqdvzOB32I64trauzuxaXOh+K6nA82m+0xnueP6vX6n+WfJSPcWK1W/ttvf+7DcTehQ4fbcOzY7sTly5d3fvjhh0+H4fDN+Z3QlLFL/Nsu+FxCHVEtqA+C72+C3f+zNXxOdqmIhiRN1LUI9CP5ayeKInOgMxiMSCLQwr0hFNvCHSxmahH41xc31lZQDF8L9UDcFFPL7iF12XK5XAO8Xu8NMTExn8o8PUaYKSoq0lSdP58yPibGwRFyMTl0rcHgNFuthv3796c0UUCP1JgplCJki/93HeCLmVxNGK8+5IpnpEYuAT3wN1guAV1ycwWLmxgMRkMwAZ3BYDCkI2QHur/qUYkoORkk6bwppeQKnA98jd1dQeuLV3Ecl1+b84EQ4oqmloBut5toNJoWlahrqXz66aed8vJUSampU2w6XVIyx40R//vfTb1nzJiRLbELXQ5RNRzH9qDaQQEACQC6AhgBIBbAgwCsAM74n3MGvlaH4YA50Bs3llzdL+RYA13O+TAYDEZTCW7hXh9KjRsA5c5dFge6v8uWqa64CfV32QruqhWuLlst+r222+1cRUWFunXr1q4wtSBv0fNtLEajUYBa7S1yOn1F6P642SqKxEYINZlMUncRkON+PRzHr60IuQ+AdABjAYwGcA7VsVW4ipDZGuiNGwcyjSXnGugsbmIwGHXCBHQGg8GQDtbCvWGUmgwCGjlv//rixmDHQyDRg8udDzXHsAcJ4yUcx+XU4nywSLi+eIt9r51OJzdixJjpw4YNOv3OO+/sknIsUfTFZxKveScZAfc5IROoVttGADwwm6dY8/J+SgqHC53SZq1hkOoaLQewD0Bb+FwWP6HaaZEO33dhHqoTQ/lofMJArmSQ1G+U3A4H1sKdwWAwmgfWwr1hlDx3oBFzD+qyVdvSU5c4x1H7+uLWYMd4TWHcHz85CCFhvX8I9/HCSX5+vnbOnDnjdu/Yke72etUpZnPpjPvu2/r888//0fDeV47b7SZlZWXqpKQkj0qlarGvS10YjUah+9Chmas2bRo4yOVydQXgEEW8XVYWZ4uLs06ZMiU3DMM01+si1fdRoAi5Er4Y6Wv48hppAK4DMA7hK0KOpoLgaBTQ5WrhzhzoDAajXpiAzmAwGNIRyk0Ya+GuzLlfFghQStWBBE8Dzgd9LccKTujkByd4glsCNvP64i066bFo0aIeZ8+WtSss3JL0+OOZR7p162a/kv137doVv2jR4qsXLXpje5s2bTz1PfeRR54cptVqhEWLXt/btLNuHlavXt2uoIAmejzH1OfOPa2vqtKhvNwe63JpyQ8//NIjTG3cG0Kq60mu5EOOf9sCX+I2IKYPg89l4cCliSHrFRxbzmSQHGNETYJGrrH8DnSWCGIwGJFCwIEeyvMis/qw6bCYKfBAdZethoTxhrpsVXIcl+ePmS4pKI62LlvhwOv1kok33ji9NDe33XOUkjQAX5eWmv/12mt3UEoxf/78BkX0Y8eOGd94440Bpw4d6qTSaj3Dx449+fTTTxOtVnvJ85xOJ7dgwYI+e378safXatXrEhMtoyZNOvbkk09mRFoB8vwXXtj/aG5u4v2Zme175OSoigQBhUZj+V8ee2xb+/btnWEYorkvVKnvn60ATsBXiMwDaI9Li5CBS4uQ867gnJiA3rhxAHkc27J01GJroDMYjIZgAjqDwWBIR6itCAGWDIpK/M4HXS0tAY2CIKTZbLaZQc6H2tYXDxbCS+pwPthbslOhBi3yvXY6ndznn6++htJRxOk8oHv11VcHffzxx9uv5Bj//vfiq3fvzuv91ltvlb366quH/A9fNt+tW7ead+/O68ZxlO7atStz2LBhFWGZhIwMGDCg7Pbbs/dTSonBYEhq1arViLy8vCNudwdbcnLyFRUe1ENd13SLvIaugNqSGxYAB/0bB59LPZAYus3/nAJUJ4bOoeFkQjQkaaJVQJejhTsH398QBoPBiAQCa6DX+zfef78b6fcBjSXq515bly2v19sVAOx2+1QAMSF22SrmOC5b5i5b4abFvdcff/xx55yzZ9uvB+h4f+z5J0pxF0CWf/jhmHnz5v1Rn7h9+PBh06ypU6caS0sTbiEElZRiXVZWh6xTp2zvffTRJfdFf3v66SHnN2/uP5Xnhe5arfvghQtJP3zwwSin06kKRahvSbRt29b15Zo161euXNneYrHc1kenKxs5cuSa7t2722QYXsr7W6mv0driAAGXFyF3hS9mGgpgFHxFyFkAMhFaEXK0xDPR6ECXs4U7E9AZDEadMAGdwWAwpEMAa+HeEBGZDKKUEkqpIYSWgEZc/rfWDV/1tOhP6OQHO8WDfjqjzPnQYkX+RYsW9SgqsiZrtfOdHs969ebNbw7KzMzcH6oL/eeff046cODsVR5PX/W6ddv7P/bYheOxsbG1Pvezz77q6XT20FLqxqefftFz2LBhO+s67rlz53RhciaElfT0dEt6evoRAPB4PJ1cLtcIvV5/kuf5suY+tzDQ3GsFivAJ5OcA/ALAgOrE0ED41lB3w5cYCgjqlTWOwdZAb9xY0dTCnTnQGQxGJMFauDdMxM69CV22RAAuwFc8QQhpqV22wk2LjJl27drVzkyIONbX7hgAwBGCqZTS78rKEnNycvRdunSps5X2awsWDE4oLU383mCwJ3McBYAdHo9q5s6d8Vu2bLFNmTIFAHD8+HFjxvbtPedqta7JcXE2ABhjNDpiSkvjvvzuu37lf/1rRkJCwsVCCLfbTXbt2pXA8zy95ppryluiQ12n04kzZsw4a7PZHBzHlev1+nCK5819vTRn3GQBcMi/NaYIOZo6akWjgC5XC3cmoDMYjHphAjqDwWBIR4OBfFCwH5EJkTBAaVAQ3txQSrmazofafsLnfKgZnTuCnA+lHMedrekW9zsfPDab7QGO4/L0ev06+WfJCCbgPgduAc/3Fglp63I6P9JfiQt96dL3+7tc3TRm86LyioopsW+99Vbvv//975c9b+vWreZDh0o6x8XNsYqik+zd+3rXXbt2najNhb5mzZrUhQvfHvX880/+Mnny5IIwTFVqwhnc1vedIGfQLgVX+n1nB3DUvxEArVGdGLrF/1gJqhNDZ8N2pg0TTQK6LA4HP3K0cGeJIAaDEUlcSQt3peawWpSAXk+XrdrWF290ly23293b4/HcodfrVxFC6l0iKcpoMe91gISEBHsVpVwpQJODHs8CiJrnBbPZXO/7k3HwYNq9HCcExHMAGK5We3s7nfj999/VAQH9wIEDiRqHQzs2Kak8eP8xRqN9hcViPHr0aOx1111XBgCffPJJx8+WLbvWXVycQAlBTGpq8cPPPLO9hcdOcsVMgd9HugM9VOorQh6A2ouQ5SygjRYBPZADk0tAZy3cGQxGs6PU4IPBYDDkIGQHuv+mTYnIkgyilKqC3eI1BXJUOx8MtZyfLSjBU8Bx3Gm/EF7T+XAlN90tKgkmIy1uzgH3uUbzuAsAOC4ehMwWN29+c1BGRsZ+s9nsSUpKqjMhFHCf6/ULXDyfIvL8ncK6dcvTH3/cgsTExEueG3Cfm81DqwCK8+fXJNfmQhdFEcuXr+yXn5+Ysnz5V/0mTpxY0BLdFBLSnOK41MmHphyfArjg37YD0AHoDF9iqBd866cHHDmd4BPTS5twrvURjQ50udxrcrRwZw50BoMRSYTUwh0trPBWZuSKmZrSZctFCLECCKwpHo4uW4FYmURZZ66I46GHHjr++SefTHjE61W9Syk1E4LtlOINQuiAq68+lpGRYUxJSXF16tSpVhc6x3Giu8Y1LFIKNyHg+eql6uPj412iSiWc83hUPXn+Ygx23uNRCTzvTUpKcgPApk2bUj549dUbJzgcuqlGo81DKfkkOzt14fPP39KhQ4dV/fv3r5LopWBcitQCfWOPH0oRcuBa7Qpf3CTFEg9yxTNyFQNHXYEzE9AZDEZDMAGdwWAwpONK1kBXakag0ckgv/NBW5/zAdXCuLbG7gIudT7k1nQ++H/apFpfXIEJwBbpGF616turBUHNeTz3azz+FA2lLuJ0Ut3jjz8+TqNJED7++O3NdYnoAfd5YuINlQBgNP7FXlGxOva7777DzJkzLz4v2H1OiE8Mj42dYqvNhb5mzZrUzExL+5iYP9tOnPi8w3fffdfmtttuuyDZi9A0mus6bpHXU4iE69ydAE74NwBIBtANwPUAegPoC6ACPodFJnzrBbrDNHY0CuhR08JdFEVCKY2UdV4ZDAYjZAE9hOdEK02ae1CXrYtxUkAghz9eChLGa1tfPCCMl3Icl1MjXgoI41I6xJX0vrfIe9y0tDT7s/Pnf7PglVfu+MHrVSUCYiHAJcbHl6usVt2CBx64VVCrhdT09JznXnhhX8eOHS9ZhqrvsGEZq3/44ZrpgsB14XkRAL5zu9WneJ7OHjPm4v3phAkTij5cvLj43Zyc1Hlmc3mqWi2cdrnUn1gsMSmDB2f06tXLCgBffvZZ7552u+5Vs7mC9xdXpGs07tvKyuL/85//9HjzzTf3yPjyKBG5PpPh+DzUVYQ8GkASgD/DJ57noNqdHq4iZLleJ9bCvZEwAZ3BYDQEE9AZDAZDOkK5CVN8C3fUmLvf+aAPwflgQu3riwcL4wV1tAR0NLOLoUUmRqSkpbpGpk27c09mZuaZmo+7XJ1UGRklbex2vf7zzz/v/MQTT5yq+Zzffvst8cCBs1cJQm9VefmLcYHH3e4EzVdfrcPtt9/Op6SkAADWrv22q92u1blc/9FUVf0nBgAoFUGplnzzzZquw4YN2w/43Oeffbaqn8czSJWScm9pYeEx86efruw3efLkCwpyodcXLMvRLlBqB7pUFMOX7LkewDoAVlQ7La6G7+/NWVQnhoqaMBYT0FvwWKIocpDPUc9gMBhNJdTvKxGXL6GkCPwFvZfdQ/jXFzfWVVAMIPCzvi5bFkLIhaA4KThusl1hl62wIlUhM6NxPPXUUxm33HLLm++++27v8vJyA6UUjoMHO99ps+mvNRod59xu9Ze//dZr/lNPGT5ZuXKzSqW6+P69+OKL+6YdOtT5pry81iMoJRUA9nOceN3EiSUjRoy4GNNrNBo69+WXt/3fvHnj7ikoSEqklJTwvKhOSzu/4OWXdwWeV5yba56sUgl8UIyp4zgMIARHzp1LkOs1uULkDojluL+NxBbxgSLkjv7/r0Z1zHQ9gBtRXYR8GkA2Gl+ELKewHU3jBMaSq4U7i5sYDEadMAGdwWAwpCOUG8vAc6I+GeR3PsQEJ3YA6ERRTLPb7ck1nA+1rS8ecD6Ucxx30TEe3BKQ47hwuSulRqkOmhY357lz556s7fEFCxb0OnToXGe1eqhnw4ZNPf7855Lsmi50g8HgTU9vf8bjKeOBskv279s3fXBw0cAtt9yc3a7d4VpaCXbAoEGDSgL/C7jPY2P/x0oIh9jY/7GePDm3fQt3oTNCRy6BXgBwyr8BQCKqE0Nj4EsOWVCdGMqCL5l0JeMwAb1xsBbuDAaDcSnMgR5EbV22BEHoDIDY7fY7agjjDXXZOit3l60wo8RubS36fenevbvtzTff3ON2u8m0m2++dTIh5N7k5AoA6KHXu1PVas8zmZltN2zY0GrSpEkX1yLv1KmTY+369V+9/fbbvQ7s3dtBo9N5HpowIXPmzJldCSFdg8cYPnx4+Zfff//t6tWr2xUUFMRM6Ny5asqUKXk6ne6iyBWbklJ5PCenrUgpOH/MJVCKEwASU1Is8rwaLYLmul7kWgNdjrip2L/tBKCGbymsmkXIuaiOmwobMYbURKOAzlq4MxiMFgET0BkMBkM6GqxiDEpcRGxSwL++eCjOh5iau/r3F/1JnUKO407X0RIwGm9oI/Y9byQtOhkUTF5ennbz5iPddbpproSECZaCgt+SanOhDxw4sGrNmq831dyfUkpsNttgtVp98bqdMGFC8YQJE4rrGzfgPne7B6gSEvq5KPVCr+/vqqpKN0WAC13OSnQpx5MzKSAFdb0+ZQD2+DcVgA6oTgwN8D//PKoTQxdqOUbNcZiA3nikFtA51sKdwWBEECIUIKA3scuW4D+GqYV32ZKKqJxUfVBKW2wHLwC4cOGCVigvNw2IiblkzfPeBoM7rrw8NisrKxZAQfDvWrdu7X7llVcOATgUeMzhcHQVxcvTJnFxcd7Zs2fn1DX+rXfccWLJoUPdFldWmv5sMtk8lJIPqqpicvR65ytTp2Y0dX4SEs57wIYukEh2oMtBba+PB76lrzL9/0+Eb430bgBGARiPKytCllPYlsNFHVVdu4CLSytGY76RwWCECSagMxgMhnSEcrPXYlu4i6Ko8YvhtSZ5UC2M62ruikudD+dqcT5YOI6z2+32hziOy9Tr9T/KP8NmJZIDzabQ4q7z2vjss8+6lpQkxLVte3MxzxupWj3Bs2HD2lpd6OFk9+7dCbm5ZSmEHEVR0Y3mwOOEgObk2Fvt378/bvDgwZVSjV8XJ0+ejOnQoYPDYDBcFpT7A05G6MjlQK9vDC98yZ4sAD8CiIMvMZQG4Fr4HOp2AGfgSwydAWCrZZxoEtBlcTigek6Sr4EOlghiMBiRQ6gFxS2yhXttXbbqEMYb3WXL4/H09ng8kw0Gw/IIcY6HCyXNtSYtuqjTbDZ7YDA4T1sshmFG40UB8ZzLpbLwvJiSkmKXcvxp06adz87O3vbZl18OW1FVZQJAhfh4y/88+OD20aNHh2v96pCgXkqI6tLPZW2PRRnR4kBv6Phl/m0vGleEHG3OcLliJkA+AZ0Hi5sYDEY9MAGdwWAwpKPFtXD3twQM1fmgrrG7B5cK40V1OB/sV1At3yITYTIQ0Q6aaCbYfc7zRgoAZvOkyoKCzea61kIPF4MGDar429/u/dnpdF52f+ZrF59eSwt4acnLy9POmvXYpEGDeuS8885buxreQzYi1YEuV3LjSsaoBHDAv3EA2qE6MdTX/5wLqE4MnYc8319yO9DlcG0E/t4xBzqDwWBUE2jhHsrzZLt/bmKXLVtQnBSOLlvBRdfRLMrVJOK7tTWCiHh/jUajkD52bMbaVauGtq6sNIwwmRzn3G7VstLSeNqhQ8GkSZMkX3rq+eef/2PmzJmnN23a1IbnefGWW265YDabJSt2rg1PgUeT94+8gYl3JWbGjo4tBQDRKXJ5/8jrp0vTVSTPSs4Jeno0XsdSr4Hekhz0NYuQY1EdM9VVhBxtArrcLdzlis+YgM5gMOqECegMBoMhHaHc7AXamDcpmKKUElEUY+pyi9dwPvA1dncGOR8qOY47X5vzgRDikqCNnJKFZKXNOyKSQevWrWtrsxGdy7VGc+7cmuTg323btrfzE0/41pQuKSlRh9uNrtFo6LRp086H85hN5b333utRWKhP2r79pO7w4cN/pKeny7GmYH2fDaV9bq6UpiabAmv85QL4GT5RIOBOvxrASAAu+P6OtIIvcSRVYUc0tnCXZU7Mgc5gMCIM2Vq4B60vbmxkly1LkDB+LqiwODhushNCwp30V6KQHIxS592imTt37h/zS0sNC3fu7La4oCBRUKkEVefO+X/73//9zWg0CgBw+PBhk91uVw0ePLhCpVKF/f4nNTXVNWvWrJxwHzdU+ETeo0pS2Ys/KE4HcNg4zFie94+8fu6z7oT4m+PPNtd5+ZHy/jYaPpNNnUMVGi5CrvL/rgN8RchSCcLRKKBLPpbX6w1cAyxuYjAYdcIEdAaDwZCOBm/2gpIrtbqwKaV8Xc4HAMGJnhhcHgDYgoTxYo7jsupwPjSnS02pArpS593i5zx58uTzJpPJXVtRS2pqqh0Ajh07Zpw1a85t06ZN2fvUU0+15DX2mkReXp5248Y9fTSaqW6r9VfDBx981LseF3qTglsqUhDukpec1vIYAMDj8WDFihWdbr/99nMmkyncwW6kO9DD/RmzATji3wiAVPiSQtcB6O7filDtTs9F+BIQ0SygS+qm8DvQWSKIwWBECiGJw4QQkVJaV8wESqkhOG6qraAYvmLi2rpsBQvjRTWFcX/81JzriytVQA9LsXmE0uK7DRiNRuHNd97Zdfjw4T+OHj2akJiY6Bo/fnyRRqOhW7duNf/f//7v6OLs7LaglBjM5vK/PPjgjjlz5pwJ9fhut5uUlJRoUlJS3FKI7+GA03A0dX7q0fxX8vsWLSvqX7SsCERFxJSHUg6ZhpvKa9kl3DFTrY+JoogzZ86QX3/9tXVpaWmfkSNH5l9zzTVlTRm7rlOS4JhAy2nhHip1FSEPg6/geBZ8RchZqI6bwlmEHG1rrQfGknROHo+HCegMBqNBmIDOYDAY0hFwU1z+C9/64kZRFOMBQBCENIfD0aoW54O+lmNagxI6ecEJniBh3CaB80EKlCokt8gEgMRExJxbt27tbsgFvnTp0j4XLhiTv/56w8D777//dCgirvucW1exviI1aUZSDqfjRAAQ7SJX8mlJ54TbEs6pW6vd4ZpDuHjvvfd6lJXFxSYnT6+wWlvrt29/o0ctLvQmf35LV5a2s+6ytmn/Wvt9nIajAOA87owpW1I2MvmF5D3q1mpX8Hg7duzA8uU/DbPb7eo5c+ZkNnX8ZuDiZ8HpdHJPPDF3xKhR1+ZMmzYtNwzHljLZRAHk+berARwEUAifoN4PvtaFHgDZqE4M1ZY4DJVoFtAld6CzFu4MBiOCqDNmCqwv7i8kjqOUGpxO56g6hPGa4nooXbYsHMe1uHuwWlC0gK4wIm7O6enpluD4ICsrS/+3hx++vWdZWfwLWq3TxPN0ZUFB8nsLFtycmJi45u67786r73h2u51buHBhn32bN/f0Wq16TUKCZdSkSceefPLJDI5reau/cRqOtnmmzR/Z92ePBgDjUOP5OsTzJuEp8ajPzz8/JHlW8h/GYcYKAKACpcUvFw/U9tKWxE2NywV8rtr58+YNtpw6pTYWF3dyOhzJi9audW2aMOHwiy+9dDBMr6Es30XUS2nV2qr2xuuNF/h4/uK9rXWTtZW6i9qq7aa1NeHwUt7/B4qQYwAkAvgU1e70if6xi1EdM51F00TcaHWgS5rT9Hg8gQ8DE9AZDEadMAGdwWAwpIO6XK5uoii2qyXBowl+oiiKvQFUBQnhJfU4HyIuqK4HpQrogHLnHdEcO3bMuHXrsd4azVR3YeFa87Jly7rNnTv3ZEP7ecu8ameG01z4TqGm1SOtTkEECt8u7O4p8sR4y7yFLU1AD7jP1eoZXp6PF02mibbCwq/NDbjQGwWn5QTHH47Uc3PPDW7/evu9rkyXrmRBSX+iJcVEfWkhkMPh4Nat24aSklYJmzbt7jl9+vTs2NjYcAqFsjrE16xZ0/7wYXvXwsJN8RMnTswLo6Ne6r8TBD6x/A//BvhaugcSQzfCJ2SUoToxlOPf50rGAORJ0nAyjgOpx2ICOoPBiDCo1+tN8nq9vURR1KCBLlter3dwBHTZCjcXndjN6IJvTpQ46Yid8/vvv99TX1YW977JZEnkfMWxw9RqT1FVVeznn3wyoCEBfd6zzw7O27RpwDSVynuVTuc+WFBg/v7DD69zOp2q+fPn/1Hfvs2B6BS5/AX5fQlHKFER0brL2l7fV18WWBM9iCa9p4QnlHoon/9K/vDU+ak7NP01pOztso7Oo05BN0BXGHjel19+2fH81q395nXsSEd06GBDRUXpD1VVxnc2bhzw/cCBhbfeemt+U86jBpI60IUKQe066kp2n3bHJz6UeIKP572W9ZbWtl9sHQ3XGs43UUAH5ImZKIB8//YrfMuDdAHQDb5W79eg6UXIcgnbcsVMsowV5ECPBPMRg8FoJpiAzmAwGNJBvV5vZ1S3BbRwHJcfJIpbCCFWp9P5sEqlWqfT6Q429wk3A1ShLfmUWjgQ8XNeunRpH6u1TYzZ/HBZRYUz/ptv1g+YM2dOZkPCpyHdYDHfY84o/bS0e8Gigh4QQbxlXn3SrKST+l56q1znHyrvvfdej9JSY1xi4qhKr7eYAwCt9gbP9u0f1eZCbxIJtyVcALCn8N3CIVl/zhov8mJ7PoEvSX4x+TeVWXWJ4Lpx40ZzZqYbcXH3Vl248Jb5yy+/7BxhLvSLyQ2n08n9978/9fZ6+/H5+X8krVy5stN9990XclvLeo4PyJcMCqbQv+2Ar0isM3xi+lUAhsBX2X8W1Ymh4hDGQC3jSEFUtXAXRZG1cGcwGJEEFQTB7PV6exNCqurqsuXxeAaJotjBaDQuae4TlpugAuqIv5e+QpQ474gvls/Ozk5MB0hAPAcAnhCM4Hnv+7m5rWrZ5eL7e/z4cWPGr7/2nKvXOyfFxdkAYJTR6IgpKYn74vvv+5X/9a8ZCQkJLaY4RnSKXGDN85SHUw7GDI6pyH8lv29gTfRaRPRGo0pQeTv8u8PvuU/nXpv/j/wRfGte66n0xMXfE78p9vbYi0UJO7dt6zxEEPjhiYnIO31a566qSrghPr5qW2lpwtaff+4UJgFd6s8kAQBVksqdcH/C8fIPynuVLS3rqUnTVDj2OFL1Q/T5wXNuyhgSU1uM4QRw3L8B4SlCjlYHutQCOgdfTjLiv3cZDIZ0MAGdwWAwpEOMiYn5EdXuvMsI3Kcp1UoA5QrJSrxBj/g5B9znWu2TLo7TwWS6x1JYuCFkF3pM/5gq6qSZpV+WdgeANKC0BAAAIABJREFUpJlJJwx9DC1OPAeA33/f30WlglBVdb8x+HGO4/gffvihQ3p6es3vtSa9vwm3JVxwnnGerNxQ2YvEEj7l/1KO1BTPLRYL/8MPv7ZxOofhzJmzXfX6rnYJXOhyJAUoAHzzzTftc3NJq+Tk2eXl5etM69Zt6P2nP/0pp4kudLm+Txt6ndwAMvwbAJhRnRgaC2ACfOv+BRJDWfCtC1hzDDQwTriQaz0/WeYkCAJhAjqDwYggRK1We0Kr1e4FUFHXk7xebx8oM24AlCkkA1EQPygRs9lsywSom1JogtIcJ0WRNyYmVta374EDBxK1Dod2THLyJQ7c64xG+wqLJebIkSNxo0aNCpso3VSEKkElWkVN8JrngTXR3efdMQBqnmuTrmlVgsrbfmH7nWemn7mZllB9zLiYoppCsttm06mdTu3GDRs4KgjxAOKP5+e3EXU6p9Nm09Rx6MYiedykbq92JtyfcLz0rdJ0xx6HQdtLWxx3V9y5MBxbjrgvlDHCUYQcrQK61C3cSYQsfclgMJoRJqAzGAyGdDR4YxmkmystGRJAyQK6Eucd0VS7z6eUAYBa3V4g5CaE6kIX7SJn2WZpE/h/1c9VqfpeemtgTfSWxL/+9cpPubm5htp+N2rUqJKg/zb5Ov7oo486ffzu8pFvpSyqiEEMAKDs3bKrkuclnyWa6iUrvv766045OTAWF5tBqZrY7V31589npESYC50AoD73+c+9KR0JrbajJzFxSmV+/k/hcKE3pwO9Pkr922744o+OqBbUB/qPdQ7ViaECyOTW9iNn20NIPZYoigRsLT8GgxE5UIR2b6zk+2elC+hKmzcQwXP+y1/+cvKhjRsHzbdYTM/ExNiMhNAvHA7dFo4Tp02efKy+fRMTE12CSiWc83hUPXn+YjFtnsejElQqb6tWrZzSzyB01Clqd4dFHfYQrvrt4jQcbftS2yPBjzWVlStXtntn8eIRuWeyO9xCJtNRceMsvbv1ttl32s26vrp4/WD9xeIjbWKi5aeKioRrAQyCT5E9CpBdTqfelZOTEqZTksWBHsB51Bkf+Le3wBsjVAiq4DXRmzCGHDHTldDYImQ5i4GjpoW7IAgsZmIwGA3CNfwUBoPBYDSSUG/2REqpUr+PlZwIU+K8W/ycvZVeVcGbBVc5Mhwxgceol5Kz757tsvOXI32AMpSV3ZBQUjIqoaRkVIIgbOTKy92xq1evbl/L4S7OV7SLXGDN8+QHkv9IujfphCffYyp8p/Aq0Sm2iM//4cOHTWPGjJuya9eu+P79+1dNnjy5oLYtLi4ubG0TvV4vWfbmf8ZkZdk6ripdFdv1q64/JD6UmOM67jIX/r1wMHX7lniwWCz8+vU7elZVtdKUlGQCcFOglCstjUvctGl3z6qqqnAVhcqxBjr95ptv2p87R1olJt5ZBQAaTWuB48aJ69b91ttisfBhGKelCejBeAGcAbAJwBIAbwJYB8ABYASABwA8DeA6//O1TTrT0IiqFu4sGcRgMCKMUAV0EcrNYSlZSAaUNe+Icd1XbKhIKfm8pAMVq0/ZftRu6rK/i/nBx+b++L3JZB1js8UMsVhMr/O8OOzWW3c+9dRT9XbtGj9+fJG6XbvipeXlcfkeDw8AmU6nernFEpPSq1dujx49mrrmdZM4efJkzN133319j86dH+3RufMj06dPH5udk62v+bw6xPNGXccrVqxoN/fRR/8n4eSprm+7J2hudHbUf1r8ZtKTZY/ouCTOVfJ6yXDHXsdFgfnEiRPtTwPYAGAvgG0AvoLvRvv8mTOdGnMOtSB10e7F4wfWPNcP0ecnPpJ4hDqoumxpWU+hQghH/NeSYyagugD5CwCvAfgcvtbvHQDcDeBZALMAxMK3trochQ1yFf/L0sKdOdAZDEZDMAc6g8FgSEeoN2JKFpGVOveISYyEkYiYM6flRKIiYtlXZV0SpyZm6brq7CWflHQiF4jpiRn3/1auK7/sc81xHL3lllvqXUvOutea4Cn2xCTNSjoZ1LY9o/Sz0qvsB+1xxmuM5fXtLwcLFrw2+Nixom6vvbbQtnbtmk1yjPnuu++m5V2wplCMpN+UbYp9wvsEiR0eWyXYhYLyD8vbus+6DdpuWlteXp6elgmmyooftL7c+QgAZ+ByntG5irvGnTt3Tt+7d++wrcsuMXTduq09HA7D/2fvzOOjKq///3nuvbNvmewLSSDsOwqoIAq4QHFptWqttuKPUhWXurTu1n7r0hat27e1tLUiVmtdwC9aRVyoLCooKCCbLAEC2ffZMuu99/n9MXOTIQnJJJkZMnOf9+t1XySzPOeezMI95/Occ/SS9NeOTRayj6+r4zNXr15dvHDhwop+rj1YK9B7wgHgm8jBAyhGuMpibOT+nwOoRkelRXUcbSskW0Bn1RQMBoPRAYuZekeVAnrU7HfGIESwC0HPFk9B87+bkXVt1nHfXp+l9e3WYZoijfumG286fMW1V1SsXLmyxO/38/PmzauZOHFid9fqJ7zGWq2W3vvII5v+8MAD511fW5ttp5Q087ysHTmycumjj36ZJNe6pby83Pj9BQt+xjsctmsohQTgjbVrZ16ybdvoTzZseKmoqKjzOKK48Odnn519uigK7wr2UK1YqsnVfBQsoQfx/YM0b/P1m6tmOGa0eTd785Uq9FBLyLYAkzEEu/Ba5M97BgQ6GhPJI4HvkrExNW6I9aKmbWNbieEMQ43Stl2Zie5+3z0k46cZFQNYfrC0cI8VZROyshE5A8BwhOOmYoRjp19F7i+P/OuNk22FtGrhLopiMjcEMBiMFIUJ6AwGg5E4Yr2wVHsySHW+R5JBqvMbKeAzp+fknMU5RxqXN5Y1/7t5OKflJDkgC1mXZx1dePpCV3/Xtc62NhsnGF3Rc71NU0wuXbHu286zvk8F27dvt37++Y5JsjyN27r16/GbNm3adu6557b0YYk+B9KiKJJ//vPNs2Rurmy0P+5wuc8zP/nkk5Mff/xxYj7f3Gw8y7iPM3ESAIweNdpz7emXHr7z4CMjOfIYCLdYpHIjkel5giD5uHFjx8VLPE9GBTpmz552eMyY+vpwLUg0pZgwYcJANlMkcwZ6IpAAVESOcgDXA/gEQCGAMwHMBuBHR2KoHICnm3X6SsJbBEbZQaJtMQGdwWCkGKyFe+9QAKCUqs1/VW4ciDDofTafZXYAgOMDR2nt0lqLHJB57RCtO/v67ApOy9Hc3NzgrbfeWt7XdWfMmNH62nvvvbNq1ari+vp647Bhw1yXX355tVarPaUbKh577LHpksNh+5LjpJLIKL5bKcUZTU1Zv/vd705ftmzZlhiW6ZMPoiiSY0ePFv+CENnEeVCmeSnEERGzwKNUkuiWnVvslz91+cfEQNqv+wqthR7amJF3DXT4FbaBgqetOAvLYIFJWx3sm9cnJSkV6EKeEMy8OXO3dqi2PWjSFGv89iX2vXwmP9AYOhmfsUTGltGbkG9EuLV7I8KC+qTIY6I3Iddg4GJxsmKmpNiKVKCzmInBYPQIE9AZDAYjccR6car2doSDPjmQINTmd8pUkHB6Ts6+Pvto7dLaibJP5kzTTPWm0039Fs8VuhPKB4N4DgB//ONT0wKBfJ3ZvMLj811sfPbZ56afe+65sVShd3kff/nllxlOp1M7f/78hp6euGzZshE1NZ48veE2P8flUo67hr733svTfvWr1m9tNhsU8RwIt0F8ZvOzYyktJhr+B5BoG8fBAIor6M7qvxft2bvHcpKqlrjx2WefZa1cuXLMU089tXkACTwCgN500019Tib2YX0gtaoperIBhNsUbo78XoSOOYA/iNxfj47EUCX6JxynVQt3NgOdwWCkGDJiiAkibVbVHDMB6osfFNTkd8rETEBYRPeX+63+A347ACji+UDXtVgs0qJFiyoGfIJxZPc33wz/PqVEEc8BYDghmEcp+Wbr1uEAYhHQAQCtra3C66+/XlpdXW0pLCz0XHvttRV2u73LmCyO46hWowk1+P06AOBI+CEBSuGklJhMplB0zAQAix5c9Mmjix8sewHTySU4CxZosAkm8iK2YMTMKTv76X5MHDx40PT8889POfzdd0UGi8U3f8GC7xYvXlzOcX3+6m6PaaLFcwVNoSYe1f6pVoHemx0XgP9GDjM6YqbuNiEfBtCf2DnZFegJtcUq0BkMRiyoNfhgMBiMZMAq0HtHrb6nVGJEbVCRkpY3WkqU373ferOjZ6IPBnwHfCbnx86c6NtCTSFN6+rWgr7OVFeqz3n+RpHjbFQQbg5t3frd+E2bNmX29bz8fj/3yCNPzXr88b/MaWho0J7scUr1OaXnEUGYJAGAwXCDz+XSmZcvX17UzbqksqphCEgzQvIMyPJkXqJTeIp/QZJ4/umnnz6jr+d6EroN1GVZxjPP/Pnsdev2Tn3ppZdGxMlWIkhHAZ1G/VsFYAOAFwH8EcDbAOoATEG4Wv1eAD8GMA3htoZ9IRnJE9bCncFgMLoSqzis1rgBUK+ArlTen+rzSDqp0m3Au9trCZQH2q+5Wt5qKY6eiX6qkf0y17q6tSDUFNJE3+78xJnj29+3+I7XaCR3N9dwHoBqNJou4nc3EADYtm2b7ceXXnrFu0899b2a11+f9d7TT8//0UUXXfnFF1/YOz+B4zhMnTlz94sA2R7eIIkApXhUFDWtPC9fddVVtZ2fs2DBgvrZC+d/8JZmN55BNh6BjTyOr6hlbMGBV155JV5jurpc027bti1j4Y9+tHDPe++dc9ahQ2VZO3aM/9vSpT9csmTJnDjZjDfpJqBH2/EA2AlgFYAnASxHeJZ6BsKbkH8JYAmACwAMRXiMVn/sJJJkCOisAp3BYPQKq0BnMBiMxBHrxZ4M9SVDFNSaCFOr34PeZypS0vRy09DA8YDV/kP7EcM4g6dxeWOZMhPdMNrQdqrPEQBkjyz49vnsVKIkY0FGQ6gppGl9u7WY8ITSECXQx76WUn1uNF7bBgBa7VVBr/evpj5Uobfz2muvlRw7RvMlycQvX758xAMPPLCvu8etWLFiaG2tK1eWyzmP58r25JUk6YVVq9YU33bbbUSr7dDf9Xo9veOOm1a31bRNNbYahzZWNR4AgFBRqJlmUO/tt9+e0GqK9957r/DgwabSQGC47q233pv2s5/9rLyfVejJSjqkg43exGYvgD2RgwDIQ0elxUWR25rRUZ1egfDswL7aiSdJaeEuyzLAqikYDEbqoLRw7w21x0yA+vxXq98pgXe3t33mefb12RXe7V6b4wNHqTITnXCn/mWjIUrEFlHX+nZrsf0Ke6UmWxNyfuTM9X3ny+Az+D61Mz/3vPP2vr58efEGWSZzuHCV/YeyTNYDWDJ/frcxT2dkWcbjv/717BFVVbmP2e3OIRqNVCeK/MO1tTlLf/ObOe9+8snqztXaTz/99KYfXnpp0flVVYWjZZk2UYpGjpN/fvPNOyZMmNDtjOvfP/z7HY6ZjgWbXt9U52x0uhbOuunAab8+7VsiJO41eeIPfzinyOGw/Uev92VG/j4vBQLaX3/66bR169btu+CCC3rsTtaJZF2fp0PMBPQcYyqbkJWNyAZ0zE6fAuBsAEEAR9ERNzn6YSfecEhwPBMKhVgFOoPB6BUmoDMYDEbiiPVCjEK9HUHUKiQD6vN78JQi9IDoEAWxRdTbL7MfVdq2KzPRA4cD5hgF9IT7appqcgKA+zN3XrOrWSs5JC3hCbVfZa/kLXzMu6j3799v+vzzHZNEsVRoa7sjSsgW+K1bvxu/bdu2L6dPn+6MYSnq9/u5VavWTqD0XKLR5ITWrHljwuLFDeW5ubldklNTpkxpmTdv6tfhStnor8oSlJWV5fI83+X789Y5t9Z4P/eO01+qp4YZhlWOVx1loWMhm/UKa7mhzNCltV8/6ZIUkGUZL7/82jRRnKyx2R501dRcm/fSSy+NWLJkyaF4rB9n0rkCvScowpXodQA+B6ADUIZwYmgswq0LRXTMVi9HWFzvq52BkhRbrIU7g8FIMWJq4Q4Vx0yEELULyWryOyViJgAIlAfMinjOaTmqzER3f+7Ol9tkvi8xSaLgLbyU+aPMypa3Wopb324t5u18IFQTMpnPNtebp5tjiXHa+e1vf7tjy+efj7rkwIHh02SZSoRgO6UYO2HCwXvvvXdPLGscOHBA566oKLjJYmkbotFIAJAvCNLNVqvn1srKvPXr12eff/75TdHPKS0t9W/cvPmVZcuWjdq2bduQYrM5cN111+2bNWuWNRgM2jrbkNtkvnV560g+hycXP3XxPrFB3ON62zWidUXrcPsi+2EikHi8x064pg0Gg+Tw3r0jfsvzoiKeA8BCrTb4J7/f8MEHHwwfhAJ6qs9A768dH06+CXkBwv/PnmwTcjIF52RVoDMBncFg9AgT0BkMBiNxsBbuvRBJBqnR95RJjKgNTbYmlP/L/P3RiQVOz8m5N+XGK9kQN0xTTU7JKWm8u7yZAJC9KPuIYBNiaR/YsYbJJJ155vh9fr9fE46PFQi02rFiRkZGbzPa2z+/SvW53X61m+ftUmPjGvvJqtCnT5/ufPnllzd2t6DX650RqaBth8oUbRvairTDtT7jbKNMOCLbF9kPt65oHd62vq1IP1nvjGOVywmvs1J9rtc/4tNoxomUnkcGWIWeSJL1fTrYBPTOBAB8FzkAIAcdiaELAXwP4cqKcgBHBmCnryRlBjpr4c5gMFIM1sK9d9QqoA+266xkMuhfa/vl9loqUhIdI5nPMjtM00zOwRQ3KSJ64/LG4bJPFgzjDa19Fc+B8Fz2devXv/Hcc8+NXb9+/UiO4+jdF1xw4I477jgQa0zgcrl4XpK4QuHEmK1QECROkrjm5mZdd88zm83SvffeG31ti+7EcwDwbfdlSC2SwXatjQrZQlBTpHECKHf9n2t4YG/Aop+sd8Xuda/QqB9IdzucCAbtGIZ0buEeK91tQh6GrpuQjyEcNxn6aac/JPxvxzYdMxiMWGACOoPBYCSOWBPkam9HqEbfaarMtYsnqeJzdwmfwZQEUgg1hTT+Q36r8rtnsyczY0FGX3b2o7i42L9q1aqPB3ougUCAKNXnOl1ZCAAE4WKxpyr0vkA4AvvP7Qc4C2dRhHIiEGpfZD8se2U+juL5CQtFV5+bTLO8AGAy/dxbU/Npf6vQWQV632wgTnYaI8cWABqEZ/0pgvq0yGNmAbAgnByqj4PN7mAV6AwGg9GVWCvQ1R4zpcy1dBxR3caBqG4DKUGqxE2ezZ4s5efAkYAl1BRyaLI1vW0U7oLRaJQffPDBvQ8++ODefpwGGTVqlF8ymXwft7UZf56R4Vbu+LitzSCZTL4zzzyzuacFYsF0jqlZN07nErKESxH5DBlOMzi1pdrdfCbfZ59PwgmfSa1WS4ePHXv4Xzt3jr2K0qCNhO9+MxTSNnCcfMe8eUf7uX6i46Z0iJniaScAYH/kAIBsdMRMFyA8K10CcAmAQwi3fR9QnN8DCa92F0WRxUwMBqNXVNn+isFgMJJEzBXolFK1fh+rVUAH1Of3oEukpDLRM8+zF2UfsZxjqfcf8Gc41jpyT8X5rFy5csixY3K+RjNEdLs3GNzuDQaNJkdqaNBlLl++fEQ8bPAZvEj4Ez82RCCUt/J9qrqPgfb36po1awoOHmwqlWUDcbv/aHa5njD7fO/pA4Ec3VtvvTctEnT3e/0Ekg42EpU4CyGc8FkL4M8A/hq5XQQwG8ASAL8C8AMA4xGutIgXyZqBzoElgxgMRuqgzEBnLdxPjuqE5E6o1W9GHFBmnpvPNtfnLM45TAQit77dWhxqCmmSfS52u12accklO1+WZW5pc3PGWo/H8MfmZtsLoshPmz9/V2lpqT8edoQsoYtQHkfxvFt+ec89n1VYre7Zfr/xHp/PcK3PZ7hPlrlJs2btmD9/fl0/l031Geip3h2sCcCXAP4F4AmENxr7Ea5S/zGA+wBcj/Ac9bw42+bAWrgzGIxBAKtAZzAYjMQRqzisZhFZrYkwtb7mavQ5IXi/9dqUmeeCTRCFqYITADxbPDmhplBrdEWFLMu47777pl588cUVc+bMGXBVQ3ccOXLEajAE/MBLhNLw68xxkC0WtB05ctwe6zoxtvdL2iw8QRBocbGtVpa/I1EdEwEAdrvdM9D1E0Ay5wWmqoDeGaWF5+cADgAoQUelxZSI/Wp0zAGsGcA5sRbuDAaD0RXWwr131Cqgq9VvQJ0+xx2xWdT4y/3W6Jnnykx07w5vhu1CWyMQjpdeffXV0nfffXeM1+vVTZ48ueqOO+7YW1hYGIjXuUTiHPrYY4/tfMpmC3zwn/9MfLu11WLIz2+ec/HFe+69994uY68GMV3en2effXbLC6+++uqyZctO+3j//mK9yeRbOH/+3ltvvfUAx/U55ZOs+eTJsJFKFeg9EQLgAmAE8A8AdnTETLMRrlD3oCNmOoLwvPX+kowZ6CxmYjAYvcIEdAaDwUgcrIV776g1EcaqsRkDwjrX2kRn0BbOyLV/z5immpz60XoPb+ZPCALfeOON4pUrN8zcvn1Pybnnnru6HwmMniAA8PDDD++58073N909wGQy9ScoPdlnJBnfF+22FyxYULdgwYJVcVw7WS3cE0myhO2kVGvjRH8khFsRHgXwCQArgOEIJ4ZmAJiDcCLoMDqSQ239tJUwWAt3BoORSlgsFup2u3uNmyJVYmqMGwD1CslqjJnU6HPCELKEUPb12Uej4yPewktZ12QdJ/qOytOfL1587mcffHDWWEniSgGy5ptvxq1bs+a011atemPMmDF9udbrFY7jcO+993539913f+d2uwWLxSLGOT5TSHTc0WXtyZMnu/7+979vjMPaydoUnOjq43QS0DvbaQWwLXLwiP8m5GS0cOcIISxmYjAYPcIEdAaDwUgcMbdwhzqrsAGVzgKPoDa/k5IMEltFQbALJ7T0Fp2iwJt4aTDO4+svhCMgxq7txjqL57IsY/nyV6f5fEP05eXVJW+++eaQa665pmqg9svLy41lZWVe5XeO42C32+PdSv1UkFSBPgEkswI90QyGSncXgB2RgwNQhI7E0GWRx9Qh3BK+HEAVek70JKuFOxPQGQxGqhGLOK7qmCnyr9riBwW1+p0wqEiJ7JM53nJi7NBdLJXqdI6PACB6E/J7772Xv2nt2jN/C9AbDQYfRwgqZJlcVleX85uHH5751sqVn/THriiKZNOmTVmBQICbPXt2Ezq9jzmOg81mG+jf+lTFt8n6TLIZ6IPPTnexTl83IR9GuFq9JxLewj3StYu1cGcwGD3CBHQGg8FIHDFdiBFC1FqFDai7Al2NfifU51BTSNP498ZRpjNMDUo7PrFVFJpWNA3XlmjdmVdm1iTS/mDkjTfeKD50qLnEYnnB4/P93fDii69Mv/rqq6sGUuWwZcsW+2233XvJdddd8dUtt9wSx7MF0Pt7JBnJgXQQn1NdpB+MdmQAlZFjPQATOhJDUwGcAyCAcLtCJTHk7LRGUlq4MwGdwWCkILEK6KCUghDVXUarUkCPxMmAyvyOkFCfW95uKQrVhwzZ12cfFWxhwdyx1pHr3enNzl2SezDdRPSeeOedd0YWSBL/c73ex0W+W4ZyHP0px8nPbd06FmFBsE+sXr264E9Ll87x1ddnE4DwNpvrl//zP5oFCxbE89QJYvzeTBAsZuqddBTQY7ET6yZkpTq9El3jo2S0cOfAYiYGg9ELTEBnMBiMxBHrxZ7a2xGq0fe0qYTuAwn3WZOtCRkmGZo9X3gKAMA0zdTatKJpOBUpZ5llaUq0/cGGUn0uSdN4rXZGCADKy28ccBX6Cy8sn1hfb89ateqjKYsWLdrN8zyQPu/pZMwoT3Vxe7AJ2wOlv1XhbQB2RQ4CoAAdiaFLIrc1oiMxdBzJbeHOqikYDEYqEct3lvKYZAkFgwYVC8mqep0jJMVnyyxLY/O/msua/tk0LPv67KOezZ7Mtq1teaYzTPWnQDw/pe9rURQ5A7q2tzASQiVJ4mVZRl82H+/cudP6xIMPXjLD5TLfYDJ5DYTQf7e0ZDz/6KNGi8ViuvDCC+N6/qeIZMRMAKtAH2x2+lMV3t0m5DKEY6bTAcxC95uQEx7PsAp0BoMRC2ptf8VgMBjJgLVw7x21CuiAev1OKPZL7XXGqcYGzxeegvr/rR9HRcplL8w+rMnXBONph8oUVOw6foCKlAyWqiil+txguMlHCAetdkZIFKfyL774ynRZ7l+cuGXLFvvWrUdHmUw/9tbWctlvvvlmYZxP+6S8++67hQ888MBwSmmqJ1ISSTLffOkioMfDDkV4rt8mAC8BeBLASoRbuk8AcB2A+wAoGVPrAGz1iizLrJqCwWCkGrHEBMr3tBrjJrUK6AAAlY78SqjP2iJtIOunWUeojwr1z9WPVcTzjAUZDfG2dbKYKd52+sucOXOOVRAifyKK7UVmHkrxuiQJIydMKO9r566XX355TLbLZf7fjAzndJ0uNEGrFR+32Vzj2trIqjffzInz6Z9w/fr1119nLFy48MJZZ575sxsWL8bKlStz42zvpLZTkHSpcgdSS6hvA7AbwGoATwF4AcAXAMwIb0K+E8AtAGwALEhg8SerQGcwGLGgxsCDwWAwkkWsleVqFpHV6rta/U6Kz9HV5rphOle8xXMAaHyxsaz+L/Ujo5M/jrWO3OpHqydJHomPt73+sHz5q9P8frs+GNyudbv/YvJ4/mqi1EbKy5tL3nzzzSH9WfOFF5ZP9HrL9BkZ/89NyHy6atWHQ4PBuP95uyCKInnyyecuXLXq45INGzak8vUrq0AfvHbiWX3gB7APwH8APAPgrwhXXCjfDdcDuB3ARQBGAdDE0TZr4c5gMFIRCbEL6Gq8hlba16vNdzW+5kkTJbVF2oBmiKZ9DrF5prkl3jZ8B3ymqoerpvgO+EzKbWKzqKl+tHqSa6MrK95aFzxkAAAgAElEQVT2+sN11113bPjkyQduEEXNzV6v4RG/Xz/X5zNUmM2ee++/f3Nf16upqrJNAoghSnjnCMFUrRZ1FRWGuJ58FB9++GHeNVddtWTnp5+eNaOysrjhk0/wyzvvPPOGG26YmwBzrAJ98NhIZTsUQC2Az9B1E7IB4TjpPgDXAjgDQGYcbUOWZUIIYTETg8HokVROQDIYDMZghyI2oZS1cFcfqb5buz8kxWdl5jln5EK6Mp3Dt9eX5fzEGe+d/tCP1jv9B/xZiojuWOvIdfzHMUI7ROvmzfygCMI0Go1YVOSvzcp6uTUra0VrVtaK1ry8LY3Z2fqWQCDQZ5FfqT43Gq/1ESLAZvuJ+/hxyfrpp5/G+9SV7852XnzxxRE1Na0FodBQbtmyfwj9raCPgWQI3KkubqergJ5IOw0ANgNYF/n9XYRbFI4EcA3CiaGFAGYCGHClEBPQGQxGChJrzIQYHpeOqFFIBtQZMyUNx1pHbqA8kKEboXNwek5s+mfTMNEpxrXaU1uo9XMGTmz8e+M43wGfSWwWNXXP1o2X/bKgG6pri6et/iIIAn373Xff+/7ixR9/UVpa/Xp2dnP23LnfvPT66/+aM2dOc1/Xy87N9XwH0GCnpll7g0FkFxUF4nbinfj9Y499rywY1BySZfIqIfQrSvFbgK794INzvvjii7gKjxFSPd5IJwG9P63V+0OiW55Hb0JuQnhU1nqEq9DnA/gFTtyErB2IsUgLdxYzMRiMHmEz0BkMBiNxdBGBTvY4SqlaNzSpWUBXo98JRXSGxfPotu2t77UGPV94CsCB2s63xW0OunW2tRnAwZY3W0Yd+8WxbAAwnmasy70x90jUw07pa/zhh/9572T3+f1+7qGHHpqyePHiA2VlZb5Y1gtXn5casrPPbZZlL+H5LBmYjZUrP8D8+fOJxWKJ16mf8HcTRZG8+uqbMymdDaPx9tDu3dfoVqxYMXzx4sWH42UwDUn1hFY62gE6Ni8fBrAz8nMmOmanz0W4zbsbHbPTjyCcTIoZ1sKdwWCkILEk5JUqbG6wjMtJImoX0NXmN5Bgn50fOXOj27YHq4M6ZSZ6zqKcI7wlPhuCeQsv5d+Vv6/u2bpx9f9bPwkAODMXzLs1b6+uVOf3+WIKQxKO0WiUly5duh1Ll25XbpNlGa+99lrxmjVrRkqSxM2dO/foz372syNarbbHa8brrrtu/x3r1k1+0OGwLTGbPUZC6GttbcYdGRm444or4t4iHwCqqqr0RyoqSv9BKWxR34+/ArAUwKpVq0afffbZW+JoMl0+k+kioKebHSAcN3kR3oS8GWGxfCjCMdNIANMRjneOoyNu6tPni7VwZzAYscAEdAaDwUgc0VUSPV1kqllMVbPvavQ7oT5zBk7SFGjarHOtDUrbdvul9jrCE6rJ1cR9t791trW57as2R+BowA4AOT/LORpvG4ni+eefH/Xqq2tmV1fXWF9+ecWm3h5fV1en3bnz4FCOM0ktLZdkKLdrNALq63X44IMP8q+++up4CNpd3iNK9blGc1dIozlD8Plm4OWXXzt70aJFh/s6kzBG+6lc6cAq0AevnZPZagGwNXIIAErRIaifFnlsJToSQ3W9nWukxS9LBjEYjFSi1xbuhBA1i6lq9h1Ql99JEaeEHMFvPstcb5tvawA6ZqK7P3fncAYurhWmvIWXsq/PPlT7RO1pAJBxcUaFrlSnbA4clF0GZFnG1T/60bxtGzeePlSWiQ7AE//97/RVb7xx8J3333/HbDaf9DprxowZrTc/8MCHLz733JxPWlttoJTIZnPbTXfdpb3wwgtdcTzN9s8FjVS7d24xRiJHpDtRvEn1eCMZ3yvJ+u5KRwG9s60ggIORA4jDJmRWgc5gMGKBCegMBoOROFgL995Rq4A+KBMFCSbhPnNajmZdk1XV+faMizLqE2HPsdaRGzgasHMGLiT7ZKH+L/Uj827NO0QEMqhfX7/fz/3736vP8HrzjBs37pi0e/fuHRMnTnT39Jzc3Nzgo4/e83FLS4su+naT0VSalZ01d9q0aXXRt1OZgnD9/mi3//2iq895/kwJoIJefwetqvphQQpXoadLIihZwnaiUXZhJFNAP1liWkS4Ov0wgI8A2NCRGJoF4DyEKzGUxNDhyO8nwCrQGQxGCsJauPeMWgX0QX1NncqYTje5AJwg5mqLtIGsq7vGUgNFbBY1jf9oHEMEIoOH7HjXUabJ1/gNow2DooV7dyxbtmzE1xs2TH2WEPknWm2IIwQfiyJ//Z49ox577LHJTzzxxPaenr9o0aKKK6+88tX333+/IBgM8vPnz681mUw3d37cAGMmIPIZKS4u9g8tLq76U2Vl0RWUElOkCv0vCF8oXn755Qd7WKM/JOu7KJU3NifLRjLtJKtVPNC7T503IZeg6ybkKnTETbWd15MkicVMDAajV5iAzmAwGIlDRowt3NGRwFcVhBAaqZRTG2rdOJA2Piszz5W27a6NrqyWN1tG1f+lHnm35h1KhE3JI/HOj5159u/baxWRnsoUjv84CixzLY2CTRBjWef53zw/pa7On2O1rnB5vbcZnn76mdP+cufzOw3jDG5Oy3X7ncVxHC699NITRXKRkuqnqsdp7VqYTKZ2QbBxRePQwJGAreiRom/7mRBqf9KKFSuG19S0Fsgyx/n9D+tFkfAcxxFRzND885+vz0yAgJ6sCvFEr5/qFSGKnVNVFZ4o+irWOwF8Ezk4AMXoSAxNijymBh2JoWoAMhPQGQxGChLLd5by3anGuEntArra/Ea6xMjtM8+DMp//q/xvhUwhVPds3bjGvzeOy7kpZx9KEmPXtdGVpS3S+vQj9O0bDf2H/MZQbUhvOdfS0tvzP1izZuxM2UKu0RGRI+Ei1nmCIF3il4Wda7dNxhPoUUAHAIvFIl1zzTVVAOD+wm1v/aJVl7kok4MhfH+wOqirebxmau4tubuNE409bmbugfb3yT0PPPDhXbff/v9GSxJ/iSxzBwF8BpC55523Zc6cOXEbZRZFKsdMio10imfSYQZ6NFwfbIkIV5wfAfAxwpuQhyPc6v1shCvUu2xCZhXoDAYjFpiAzmAwGIkj1gp0tYqpgLp9V5vfaVVBIjaK+uiZ58pMdM/nnnwajF/CK1QX0grZQogIhPrL/SbPZk9hqC5ksP/QXqnJ1QSbXmkq9e3y5Qo5gt9ytqW1t/Vat7VaV618fyaRvsdrNKeJGs0icdOnT562nd8uj79k/OFQVcicszjnMGfsaN3Y+HJjqXWutT6q1WIYDpToiexa5QKpJUNyrss51LiicWjLWy0TLbMsR7oY7wc6nU4sLMyqpXQXAXbBbDbZjEajtaGBr8rMzHDGw0aSSVZyI9UTWoqdwVAVnghb/fFLBnAscvwXgBnhxNAIhGcAnrtkyRKpra2tbdSoUWhsbLTH44QZDAYjScTSkUuZga62a2i1t68H1OV3esVMTlEADznv1rz9SiyRf1f+vvo/1Y+RWiQtX8K3tx+PB1988YV91cpVI8aUj8k9q/Ss5tF3jd6lH6H3+g/5jY0vNY4WsgSfeZa5pbdNvn6fTztRnkyaRLOQLWwWBeKHTDUYT6cS2ZlncXzkyOENvBQtxvvL/UbPV56s7J9kV3ZejzNwUqgmxDU901Sov1uvkf0yV3lf5UwqUo7Tc3ER8H7wgx/UZGZm/v3Pf/7zWWsPHSoZOWZM7qPz5m29/vrrP47H+p1Ih03ByVg/HQX0wVKB3hNOANsjR5dNyJs3b8ZvfvOb0KhRo5xZWVk1hBCBUhpTMQKDwVAfTEBnMBiMxBFrooO1cFcZkcr7U30ajAGQvTD7eOfbrLOtzREhHfAM3IYclMmhyw9doRumqy97pWy9aYrJJTmlI8fvOn5J4wuNXO4vcjf49/pzbBfbjsYingPA8o3LCyu9HoOZu1Hig5LGqPlx0ON62fbP469kPVvwzJ6GZQ1n+vb5skueLvmKM3Jy/Z/rR7aubp0oe+RdebfllUevRTiC7CXZlc0rm89qeaNlvPtTd0GoLmSzzLIcKXigYG882hEuXLiwYuHChRVRt58L4AwALw9k8R5IRlIg1avDWQX6wG3FQ6z3APg2chAAhcXFxXM///zz4l27dmkBvEUI2Q3gw8jxGaU0FAe7DAaDkQhYC/eeUauAruaAKS1ea32Z3jfkkSG7om/jLbxU+FDhXgDw+XxxsSPLMn70ox/N+/jjj2cIAHhoMIPMlC/Zd2nh1Q9e/ZnrU9cQwS74c5fkHoolRply+unHX9/11uir6WwCcaaQKXwtVYuT+b0Q4JkSPOB4xzEicCyQVYjCzZZzLS3+cr+x8v7KWYQQav+BvYY38yeI4qbTTa5MY6a/+ZlmTcWtFbMhgYCHPOSxIVv0I/VdxvH0gRM+I+ecc07zOeecswbhfP9DCHcnShRsU3BsNtJJ2E5mBXq8fOqyCTkUCk0tLi4+bcOGDdkOhyMbQBMh5BNE4iZKaSI/NwwGI8VQY+srBoPBSBashXvvqFJAh3r9TjmfgzVBnXe31xJ9GxUpcW9yZybaNqflqP2H9h3uze7xRxYemSv7Za7+T/XTxSbRri3RNvj3+nNsC2wVtvNtMbXkU2afB+VSzk+2U5/338Zg22qzhFH0v+VbiytMFVL+Xflb/OX+vOO/On5m3TN1o1tXt060XmA9kHtLbnl3axKOwHKZBYQjNFQXshGBSAMUz1PuPRKN7JM5uU3mo2+jMoXYLGoivyarWiMZNpiAPnhsUQDVDz300L/Wrl37h4yMjCoA/4Nw6/frAKwDkBFnmwwGgxFPWAv3nlFl9T2rvE8tXBtdWZ1v82zx2GWvnPDP7KOPPjrp448/nvEkwoPd6xDCZLqZ2/jNxolbX9g6hUqUy70l92BnYftk3H333buQZ2m8Tt7IvScS4XX/TN2zIaPmc8su9y2/ueXL4ieLt2oLtY6apTUzm99sHlJ5f+UsACj+Y/HnJ7OhGaKB/Tp7neyW9bJX1hXcU/D1AMXzU/m5GLBtsVHUdrkteTGTYiOd4pl0sqPYSoRY75k9e/bGFStWPPfTn/50tVar3QzgGQBFAF4A8IsE2GQwGCmMGgMPBoPBSBashXvvqNl3tfmdkhUk1b+tnnb8zuMXeL8Ni+hUpKTi1oqZlQ9UXhisDOoTbb/gnoL9ebfmfeLe7J6wa8yu27zbvaNsC2zb9CP0jQAQOBKwUDG2ZGpzc7NGkkK81XrYo9E8GtJyv5e03O8li/ELn1bLhSoqKkwZF2XU59+Vv8V/wF/geN8x3jLbcqjg/p4Fcd8WH6gcPgcqUr751ebSWP2jlEKWZb0oirnBYHBEIBCYIsvyMITbrPEne1qs6/eDASUF3Gvchc6VzhJFRKcyhecDT4HrLddQ2S9zA10/BpIpBqdTwglJstXXGej9ItLi5BNK6SIAhQDGUkobB7ImIWQ2IUTu5pAIIWfE5cQZDIaaibmFewyPS0fU7DugLr9TMmZyf+7OrLq/al71Y9UTlNucHzlzK++rvKD26dpxibb/7ttvT5tHCL0HgAHhXYNLYUQOOPrVd1+VgAKhupAu1vUKCwsDb73zzr9L5p779Wc6fWCDVhfKKBld9ed//OP1qVOnOnkLL5U8W7KFM3LBxhcbp0lOyVjyVMlnuqG6k5bUS04JzredOcrvDX9rmBglGPcKpZSXJCkjFAqVBAKB8aIojgEwFIA11jXiTL/fq8HDQaPzdWeZb4fP1n7bsaDB+YZzmPdLrx3J66qVDjETkJ4COpdoW7IsE47jWiilj1JKZwLIQVhM7zcsZmIw0g/Wwp3BYDASR0xtBgkhrIW7+kjJxIgaKXq46JujNx294Pgvj19Q/Mfi/zb8vWGi91tvad5teZ9ri7X+3lcYOHl35B2o/0v9vMivlDNwIdvFtqOcnpNaV7cOb1zeiJzFOUeJQHp8XxUVFQW+/XbrCvcmd2bL6pbhhrGGZk2+xuv61FVinmGuybo0qw4AAocD7UmYUF3ITP2UI0bS7e5vx7uOHNd/XTDPMlcU3le4s+Z3NROa32ieBABZC7OOy7JslmXZSim1RP5VDgul1Ipwwqfz9agIYCSAeQAOATgQ+dePQf59YTrH1OBa7Sp1rnSWWK+wHvdu9OYGjwRtptmmGk7fPlc+1RM16VaBnhRRO0JS5q3LsswhUs0ZEdMPxHH5+wFs6HTbnjiuz2Aw1ImE2GIm9Pa4NEWtArqaY6aUeq0tsywtGT/I+NbxrmMyAJjPMDfU/K5mtrZE25R/V/53ibbvcjisE6M6NARhQyvORgZayMfaj4KLsxd7G19qHJ3zs5wD+hGxVX2PKBzhf2rqU9XSaKnRfpn9qGONo4RsIQWh00JOTa4mGKoP6WiQtscxweNBw8kE9GB1UNeyrEVPQ9RV8lzJp5JbEmp+VzOj8v7KM4uXFn/F2TlOlmWrLMuWzvGS8jMAU6dlRYQ3HE8BUIvw9d5+APVI/PX6gN6fmmEar7ZM6/Ru8hYAAJ/JB93vu4v5TN5vON3giHpoKo/WAtJP2E64qB1Fwn2SJIlQSts7RlBKW+K4PIuZGIw0gQnoDAaDkTj6UoGu1o4gahbQ1eh3yvmsLdb6h/192LqjNx69sGJJxaUAkHd73mfZP+06Az0RyEGZHL7m8DxCiKwdqq3xf+cf7j/oz7POtn4dEcwPt65uHe7a6HLF0srdu9trUcTznJ/nHCEcAQjg+q+rhLfywUB5wKa0bTdOMTbVPVs34/ivjp+pzERX1qGUClJQsnp3ekuM5xiRcVmG1R/wL8i4K8NCikheW13beJ1bpyUciX7NJQBuQoiLEOLiOK6GEOLmOE753U0I8fp8vmmUUgfCO8DHAPghOmaXiUjs+2hAgbqQJwStl1uPuVa7Sh0vOUYCgGmOqUY/We+KWj+RsBno/bMDJGeeX1L+dpTSdgE9AZRTSrcmaG0Gg6FeYo2ZlO84taFqAV1lretTdtNA4f2F+wDA8a5jsvMDJ3TDdXVD/zZ0U6xt0wfCkKFDq/+za5flD5RyAoBWTIUfbqzAl3Tm6Hk1ubfkHmxY1jCq5c2WYbGOm2r4W8NIySNpc36es19fpvfpSnTe+r/Wj2n4W8PozKszD1c9VDWLaIlY8vuSTfXP1U+pWVozU5mJrqxBKSWyLJs8uz1DIYPLvjtbpPn0LEKJNfvxbHvr8tYcV7XrHK1O27mduTcqZqpWflZiJ47jXKFQqDQUClkR3mQ8BsAMAHMAOBDefAwk9juj3+9VwhGYLzLXej7woF1Ez+W9titslURLkpUrSacW7kiSnXRo4d6OJEkcgGCClmcxE4ORJjABncFgMBKHIqDH8jg1JoIA9QrJgPr8TtlkkKZAExDsQpvYLFoBwDjR2JoMu4p47t3uHZl/d/6HebfmlVc+VDml+bXmc44sPDK37JWy9ZazLa2aXM0+3XBdTJUUhrEGT8b3Mo5ZL7Q2KIkj+6X2Ot7Ch6hIScvqlom2+bYjOXflNFFQS64x19G0rGly/cr64dYrrRSAUgFhAICsu7MAAZCpPJPIxAUCl+1HtmOQ4BF0glMRxiMiuZeQnt/2lFKeEAJKaROA7wBsQrhKfRTCiaERCH92bkJHlUVdP/68CYPP4YOcmQtJAYkHAO0IbVvU3enQwj2ZAnraiNoRklLtHl2BzmAwGClCLN/3ahWR2Sxw9fqdcphOMzU53g0XEOvKdC3JEM8B4Bd33PHFDYsXj5kH0DsA4sdXeBJBGtCS4H333beNt/BS7i25ByWXJMQingOAbZ6tmrNwor5M7wMATb4mmHdz3n7/Eb+x7sm6qSAgQ/44ZKdQJGTm/zHfVf+H+pL6V+qvIhNJK9ESSyRmsgDgtLO0yD4zG0RDhkqSlEkIcelKde68h/NqeR3vjBbGI5uKe/27ReKqEMJVrXsQrkYfCmA0wnETAFyCcPy0H0A54icWDvgzSTgC3QSdI3g4aAMA3QidKyKex2X9WE4hSTaSETMByYub0qqFO1jMxGAweoEJ6AwGg5E4Ym0zKFNKTzbrN92hKqsoUFDrxoGU81mZee4/4s/P+knWNudHzlHHf3n8gpJnStYZJxvdibRNBEJ5K+9TxHMAKP5d8U4hQ/AHjgQywYUDSv3I2NoQUko5mchm43lGTUgMjVVaq1NKrfx03kopteQ+lWvVjtRODAQDAADtNC2yf53dpi3WSpGqh2NR1RBukRdzJUn6ntFofJbjuECcXO8cKLsAfB05zgcwFUAjgLMAzAbgRFhMP4BwlfpAguABJQWUmedSi6TXT9I3Bw4EMpwrnSW2q2zHOROXjOA8mcmmdKnYSKaAnhSfIv+vJur99hdCyJsAvAC2AHiMUvpFgmwxGAz10GsLd8QeW6UjahXQ1eo3kII+Oz9y5tb8rma2briuTj9K3+Rc65xQramWih4uSnjb4iuvvLKmqanp3888+eSCy5uasoA2DCsurnnhD394f8qUKS4A4C28xFt6F/QppaCU6rTjtXpZlq2BQKCjlbqVWrnJnMX+a7uVCMQo2aWzpIAEaIDM+zKDUq3kIVqiJ4Q0cxx3NFoY98N/DcdxO4xG4/o4uh59XSkBOBw51gF4AGHRPBfAxMj9R9ERNw00lh3QNW3wWNDgXuMu5nN4H2fiQt7N3nwIoIbTDM4BnlesJFPcToaNZMUyyRDqFVuJbuGeyE3HLGZiMNIEJqAzGAxG4mAt3HtHrUJyylZjq43jdx+frsw8z/5p9vHMH2YeV2ail71ctjaRc9AJR1C2omxT59sL7inY3/k2SqlGkiQLpdSqCOPeHd68YFUww3KRhY9UQJjFZpG0fdoG6w+tIDyRAES3A6w2jTMpVQ9KBYTHPMl80iBZlmWjJIVHLcfL7V7ulxCutPg/hL83SxGusBgN4AwAAZw4Nz1mUZ/KFCDt1Rwdt4uU9DZfXsHzoSdfmXmun6x36cbpnMpMdNuPbcc4Pccq0Ptmhwno/SBBLdydAJ5DeJZfM8LVTPcA2EAIuYhS+kmc7TEYDHURcwt3qDNuUquQrMaYKSV99nzpyVBmnitt24mWSI53HZN5Cx/MvzP/YKLPYcmSJYdvvPHG53fs2GHT6XTShAkTPJ0fo7RUV+IlWZatUkCytb7aWmK9wirxVl4RyzWeDz3QT9BDU6wBgDZCiBIjVekL9K6o38Mdt8xcAFk9niKN6iYRDwh6/07Yh3B1egbC8dJoABcBuBhADcKV6QcANPTFMJUoIXz/Y6ZQVUivzDy3XWGrhADq+cBDvZu8BURDZP0Efbq0V0+mSJ82sUyUrUS3cCcIj4iLJyxmYjDSDCagMxgMRuKQEXsLd7UlQxTU7Lva/E7JZJD9Mvth03RTbfZPsiuBjpnozW80D9UUaRImnitEKiAMsixboxM9ShUEAEvkX32np/r9+/0B17suASLqbJfZDkl1UqDx/sZhNERDWedlfaIr0Tl6a6keA8l+H0fbkxGuojgKYC2AfHS0LJwQub8CHYkhF3rA85EnXzNEw+nG66jS2tH3tS8jcDBgs11pOx7VUvCkaEo1bZoijVeZea7MRA98F7Byek5G8pIOTEDvmx0kyVbCWxEC7RXocU04UUp3AtgZddMXhJB3AOwG8AQAlgxiMBgDIZZNP2oVkQF1+w6o0++U8tk4xeiyfc+2L+/2vP1K2/bC+wv38SY+ZJ1nrenl6XHxlVIqUEotkyZNssqybPH7/UrMZImKncw4cROOHGoMeXzbfXzgYMCX90BeOW/jnS0vtljd77vzuCu5bzNuyNgTS0v1QUbn61sHgK8ihx7ASIRjplkAzgPQio7K9OPo4ToyVBnS+7/1ZxpnGylvCTdSlFyS4H7HPcRwhqFJN0bXZeNCZ/gsPqgboXOa5poalBjLfJG5tm19mygUCH6kR8wEJKdiO10F9GS0cOcopSxmYjAYPcIEdAaDwUgcsSY65Bgek66oVUBXq98p57N1jrUZ4Z3D7WiLtf7uqsD7CqWUk2XZrAjisix3tAeMJHoQnpsXfb1GAXiUigdCSEV09QMhxMXzvJsQEjJdb4JcIU9q+UvLWPE7sdW7wzsEMlzFzxSv15fq27o/q1MPpf2Ok+six0YANnRUWXwP4UqLWnTMTa/v/GRNkcYr1ohEbpMthjMM8G/3Z3i/9ObpxupaYhHPAUA/Xt+lFaKQJwSFPKGpv071kWRWoCeaZCVokjKXPEIqV6B3Z8dJCHkfwE2EEB2lNF5jHBgMhvroNR4ihMhA+3ec2lCrgK5Wv1MOTs/JhQ8W7u18e94v8g4NdO3IhmJ9ZENxl3hJ+RmAsdNTg4QQF8Idt5o4jjuqxEscx7kjP3vJCEK1t2mtVQ9Vza67o07SlekCni0ei/1y+5d5N+bFs3KexGHzcmf6c13pR1jM243w3PRhCMdM4xEekeVDuJPXfoTbwZ8wN53P5EMQQH1bfbx+gl5LDER2/Z+rGDKIkCvEdC3IGTjZPN98QjxGOALz+ebGfvjTX9JtBnrabAaOkIwW7oRSymImBoPRI0xAZzAYjMTBWrj3jlqFZDoAkTBVUZ3DsiybQqHQsJMlegCYcOL7XwSgCOMujuOqlLl5UYkej5JA7g3CERT9T9EusUk0uj51jQGAYcuHvW8YbRi04nkMxBpIOwFsjRx6hFunjQEwA8AchKswouemy/pJelcoO0S9n3t1vq99owFAN1bXEudETrJauCeSdK1AT8Y8v6TMDUyWgN7ZbJLtMRiM9CKWDcVqFlMp0N5hRDXEud11SqA2nyMxMQmFQoXRo6g6CeNWAJpOT22LipkqlZ+jZ45zHBfsYvAkmE43uYb8bsjG43cdnxeqD9msF1j3J6PtfAKJ9btCQnhOejmADwAUoGM81qTI/UfQETd5OBMnmeaYXL6vfdS50jkMADgjF7JeZq3kM/lQHM8/HWKNdBPQk93CPeEz0Cml8W7h3huq+pQBUKMAACAASURBVI5nMNIBJqAzGAxG4mAt3HshkiBQo+9q9TvliVRAGJUKCMkjZVAdNYO0t1K3ypJsgQhIWum8yHxwAPBHVUDU8Tx/KCrpo4jkvnhXJQSOBIzBY8H2aXzOj5yFhtGGAVeCdCYByb54rudHePbfHoSrLIYinBQaC+DMyP2HAOzXFGrA2/mQ2BCOo01zTYmogkj19urpKqCnTdVGsgR0QogdwKUAdlBKY05SMxgMRjewFu49EHWdpTbf1eo3kAY+U0oFWZYtyigqqU2yEQMxR4vjsl+2cnoOgUDghsjTJHRsKHZzHFfbjTDuiXdLdSpTONc6i5Xffbt9BaG60EFNvibe1zfJFs/6Yo8iPBe9BsCnAOzoGI91MYBLAFQD2M/pOYN+kl7y7wxPNDPONDbEUTyPPp9EwgT0/tlKmxnokU1pCRfQWczEYKQ2TEBnMBiMxBFrBTpr4a5OVOV3RBge1D5HWqpblIrxbioglJbqvNgkQvbJcL3hgnaE1me9zNqCANyBQ4Fa7wZvE9XRsdk3ZW/R6XVfcxznJoR0SShIbokXW0SNUCq0z1KnMoX/oN9kGDPwKnF/ud94/K7jc0FBhi0f9n7zv5uHt7zVcjoA5N+eHy8R/VS8pgMJ2iWEWxEeRtcqi4mhyhBokI7RFGogNom1no88+eb55jplJnocSNbfK11mBqabgJ6yLdwJIf8CUAHgGwAtAEYB+CWAXAAL42mLwWCoklhjJkCdnbvULCQD6vJ70Fcndmqp3h4vybJsBRA9ispAZYrA/gAkpwTvf71S5i2ZrUKW4ArsCbhkv1zrfMs50n6znbeOsb4WtaG4y9/At89n1o07cba2v9xv1A7R+jk9NyCRjcoUtX+oHe/82DnWfrl9l+VcS13VQ1Wzj91+bHbpn0o3JkBEjyeJ/Gy0AvgychgQvvYbDeBc2S9r/N/6ZU2RZrLcJle1fdYm8Xb+uJAnxOtvxWKm2El2LJOsrl1ACrZwZzETg5F+MAGdwWAwEkesiQ7Wwl19DPrESAI4pT7LsqztSRiPtAc0d3qaiHDFuIsQ4lTaA3Ic5/Z84rEFDgQ0Grum1f2au0wul4+Cgrg/dw8nAvFn/zYbvMA38zzfcrJzan6teaj/sD8j9+bcffoyvY/KFI0vNA737fNlF/66cIcmd2DJmoZlDeMhgxQ/U7zeMNrQVvQ/RbsAoPX/WidnXJJRpS/T+2JZh8oUnQXk7m6LE70tGm+jtZFjvf9bf0moLnS9dpiW6sbppodqQsS/w9/i2+Y7bJhu+JpwpCEO9pLVwj3VbSh20lFAT9UW7rsAXA3gFoS/K1sAfAbgJ5TS7XG2xWAw1IcE1sK9J9TquxpjplMKpZTIsmxW4qXIvyeMokK4pXrnfLKHEOKOxErHlZ9DR0OhtpVteZpsTbN0TMpuvLeRWOdZy93r3GXB6mCm8Txji6ZIYxMEoe5k5+Qv9xvrl9VPsMy0VGddm1UJAN49XnPTS01jjVONddk/ya4ciM/enV6r8xPnGPvl9l1K2/YhvxuyseqhqtkN/2gYVfRw0Z5Y1jlZfBR1e7w/vz2tF+/rWx+AbwF8K7kkne9L39VES4rN883FRCBjvVu9Ad/XvmrjDOMOPpM/AGCg1ejJrNxOtI10E9DTJj6TZTkRLdxZzMRgpBlMQGcwGIzEEVMLd0IIVds8uyjULKCr0e+406mlujUikEfPGVcSPbpOT/Up7QAJIbU8zx+ISvq4IlXjJ22pnntdLlf/p/rRwZqgxTTdVOF413E6lSgn5AqtebfkbdNM0pzW27lnXp15rO65OlPDXxvG5d6cu8/1savQu8ebY7/cXj5Q8RwAih4p2h5qDGkVoVyZie7b5zsSq3ju2ujKavhrw2klfyz5XFus9QNA4FhAX3lf5azcm3N36M7q/GdNXcRG0a8t1cq6MbotAL7VFGpGUpFOEOvEKaCYjvDc9P0Iz/87jv4LoWwGeux20iZBE2UrJSvQKaVPAngynmsyGAxGFL3+n6riNuYAE9DV5jeQAJ8jLdWtyiiqzsJ4JGYyd7Id3VLdpbRUj1SLK6Oo3ISQbj/DujE60B/QupaVLSOMk4y17g3ukfXP1J/PWTivaaqpIvvObAvlqa2n89aP0HttF9iOOdc5SwHAMMnQ2vRS01ghR2jLvDyzeqB/F9PpJlfp86UfGScYPdG3lTxX8qlumM4byxpyUCbHbz8+wzjVWJt7Q+5R5fbaJ2rHhhpDpuIni78e6HmehKRvMpE9MgC4DGcYvJyRew5AoWGaYaz/G/94SukVCG9Cj56b3t/OaukQazABvf92gARvPJYkiYDFTAwGoxeYgM5gMBiJg7Vw7x01C8lq9LtPPlNKeaUCopuqcUUYtyA817r9aQgneZRET2MnYVwRxwe005jTc3Le7XkHap+sHR+qD5loiApCvtCiydF4Mr6fUdvm7T1PINgFMf/O/H11z9aNr3u6bjIA2H9oL7edb2sayLkp8BZe4i38CUI54Qiik0O9nmOWEJAckrHitoq5Q58fup7KFMduPzaXBqhGyBICUQ9NZvImIbbMF5ijK8z9AHZrS7S7tSVaHsAwhFsWjgdwFsJVGIcQFtQPA4h1w0OiP/esAr3vKB1g2Ax0BoPBOHXEEg/JQPt3nNpQs5AMqMtvCoQ3Ccf8hPCGYkNnYVz5GZG26gi34Y4mQAhxIdxxq4HjuMOdhHEXIcR7sg3FsWKZZWkBUN7yVstIzsz5OQvnFexCW+6S3D1ES86SpN4vWeyX2WsBwLnOWere7C7SFmld+Xfk7+eMA2vfrtBdfGQYHftILSIQytt5X/OrzdMAIPeG3KO1T9SOdaxxTMi4OGNPVGV6yndV0BRqAkK+4Iz4RAFUc3qu2ni2cR2ATHTMTb8E4bnPVejYhBxrnJtO1eHpJKAnJZZBkuIzWZY5WZYTPgOdwWCkNkxAZzAYjMShBHOshfvJUauAnvKBcz84wedIS/WTCuORZI+p0xohdFRAOKJaqkcnejzdzc1LBLJP5mS/zHt3e4cAgDZP6xBbREvt0trxll9YYlqDt/GikCH4xWbRAAC6YboBzz6PJ8YJRk/xU8WfVt5ded6R/3dkAQAQDZGKnyr+1DjB6AkEAsn+/CZDgO78/pEAlEeOznPTJ0Xuj66y6GmDQrJEYSag980OkLx5fsmwwwR0BoORasTy3ahmEVmtvqsxZjoBSikX2VB8slFUFpy8pboyiupYVAW5mxDi4nneRQgZaJvtmBFbRK3YJBr9h/xFQpbg5Eycv+bxmjPzluYJJDu2t7VuhM6NdeGfNfkab7zE83hAOIIhvxuyo+qhKjS/2jzN8R/HWMkpmTIuzthTcF/Bd6filCL/JuQzRDhysuv0FgBbIocRHXPTZwO4AEAzwvHSfoSF9ZOdXzrETEB6dQdTbKVNhzBZluNegc5gMNIPJqAzGAxG4oi1Al2tIjKgXt/T1m9KKYlqqd6e6JFlOR+A0NbWdmsk0XOyluquqJbqnSsg/AOtgIgXYqso1D1bN87zpWcYEYhkOceyj+iJZBhnaHBvcg/HUCDz8swe11BmnvuP+O22C23H2na05Sjt3GNtsZ4MjBOMnvxf5W+pfqT6PAAoerDov32pYo8VKlJS8/ua8dabrRwxdGyCaFnWMso421irH693x9vmyU6ll/tqIsenAOzoqLK4GOFKi2p0VFk09nH9gZJOFejJqnBIu6RTpDpz0CSVGQwGIwZi+W5Uq4iMqOtftfmuVGOnpd+UUo0kSScI47IsFwFAIBC42u/3G9B9S3UlZnJzHFcdLYxHum15TtZS/VTQ+p/W/JY3WsYGjgVyDBMMx4RMoU1bpHV5d3kLGp9qLMq8J5PA3PMaysxzbZHWpRuuc7o3uYs5HScpM9EHA4qIvn/O/hGSUzLxNr6tk3get/ex4wNHnkQku+6cjs3XwcNBo3utuzjrtqwD8bLTC719b3sB7IwcAoAyhGOmyQBmRu5XNiAfQde56emwWTfdKtDTaoNzpIX7oPmuZDAYgxMmoDMYDEbiUAT03mAt3FVGJAmWcn5HWqpbIkeXivGolurRHRUoADcALQCR47jDnYXxeLRUTzZNK5qGy15ZY5hoqLbNtVXavmerr/9T/ehQfciUcWnGXpmXp/S6xitNpcrMc9v5tibLbEtj3XN14xr+2jCu4J6C3fGYgx4PAscC+vrn66cpv9f/qX66YZRhvTITPV60fd1ma32ndUrboTaS92ReOW/h0fRM0zjnG84ZYrP4tX68/lskPmjv6+eyFcCXkcMAYCTCiaFzAZyPcBWGUmVR2Y/1+0o6CehpVeEQIeGbAmRZBsI+sWoKBoORSsQaMwGsc5caSSm/O7VUV4TxE+KlyL/6Tk/1IzwmCISQJo7jGjsJ4+54tFRPJt7dXotrnavUMM5QrynUuIc8OmR72/a2jJaVLSNsF9oOeyu9Q3rzJ1ARMCgzz5W27ZyWk53rnKVET6TMH2bWJMmdXqn7Y91Y5WfJKZka/tEwLHomerxoeaNlrO+Ib0z2/dn7THNMCB4OGmtvr11AJcrZrrEdEbKERF/f9vVNKAI4GDkIgCHo2IR8WuT+wwjHTMpjEk06tYlHEuwotpLVTQtIsE+JmIHOYDDSDyagMxgMRuKIuYW7Smf5AepNBA26doSyLOt6EsYjVePdtVRXKiBaOY471k0FRBshhHq93h8DIAaD4cPkexd/Mn+cWUGDlNMN1bVXiufdnnfAt99nNk0xuTweD6Jm3XWLeYa5STdU57HOsTYDHTPRXetduYNJPFdmng/929A1AFB5d+V5ykx05LY/dMDvafNZZkfRo0XrqpdWX157S+0s3TjdIddq1xmm2aY9OQ/lfDvQ9ftAf33xAdgVOQR0zE2fCGBG5H4tgGwAGnStsogH6ZJsUuykm4Ce8KRTMBhUridYMojBYKQSMnr5Ho4a0aPG2AFQYdyUrLFMfSGqpfoJc8aj4yeENxRH51spwi3VlVFUFcrPkS5bbqWleigUKgkEAou0Wu16QRBinRc9aDFONLqzrsvab55udiq3WWZZWjgzt98wzuA2S+YaSZJsPa2hHaL1m2eYazIuzqhV2rbbL7PXEh2RjBOMrkT7ECvRM8/z78n/ruqhqtOiZ6JHHhaX93TpstLPKu6syGtc2jg+cCBQ4/6PezKVKJf/dP6HQpYQQjjmSDT99YUivLG4EsA6AFnoENN/ELnfCYBHeKZ6y4DPtHtYBXr/bKVNfBbZeMxiJgaD0SNMQGcwGIzEwVq4945afU+a351aqneZm4eOCojOQba3c2vA6ERPRCTva0v1lH2tQ3UhrZAthIgQTuRpC7UB2S9zoaaQRpOtCQEAp+dk0xRTzEkcw2hDm2G04YSZ54JdEAdaRSEHZUI4AuVcFSSPxPNmvk8Bom+vzwYJnDLzHACKnyr+tOr+qtne3V6b4XxDXF9T20W2BlEv7q/9ZW1Z4EDAbjzLuD/vibyvOm1GGEwV6CdDBHAocqwBUIhwUuhshOemj8OJc9Pbul+mz7AK9L6TlAqHCAn3KRQKKa8PSwYxGIxUItaYCTE8Ll2h6drKvAeS+ppHWqqfVBiPxEymTucjAnBHddiqjhLGlY5bg6qleqIJ1gR12kJtQPndPN3s7Hxbe8wUw9AqIhCaeVVmdefbMxZkNAz0XLuLj/oTM8l+mfPu9uZHzzxXZqJ7v/EW0MU0rlXovJmXip4u2lOxqGK645+OWQBQ+I/Ct5M48iqen8lmAJsjhwnhuekzEd5w/AsATegYj1WN+F1LMwG97yR7xFYyWrizmInBYPQIE9AZDAYjcfRaSRH1OLUlQxTUKqADcfA7qqV6F2G8UwVEdIcDGeEkj5Loqe8sjCeopfqgqyCJFdkvcwe/f/Aq3XBd3fBXh39KBEJlv8yVX10+X2qVzGM2jFnZW7V5Mql7qm4cCGjBPQXfKSJ644rGob49vuwhjw3ZrlRuxELGRRn11vOsazl9x3OME4yeEatGrOX0nOzY5hjOj+FPeE7b9jarbrjOK9iEfr2HAocCRuVnsV60yR5Z4K28slYy/tDxfq9ShJM91QCmAfgGYcF8DMIz0/8/e+cd30Z5//HPc9rbtmzJI07iOCEJSQgjgUASEggBEkp/bEpb2gKli9mWWSirLSVAS6EUKHQBpYUOCmUTIIUEwh6BEDIcJ7EtW7Lkob3unt8fd2efFQ/ZlmRL97xfr4NIOt1zj5bv+/0838/3ZACt6E8M5aLaiAnooxsHBRor70mnVCrFKtAZDEYxkk3cxCzcWdw0JiRLdXlB8YCe41K8JLtt7WepTggJQnTc8mo0mp2ZragIIbEcW6oXbcwEAO13tM/xP+I/quHBhv9Yj7R2A0DgicCUthvb1k65dcqzk8luPfxe2OH/i3+u63uureYF5hAAxHfHTd7feA8q+7+yJscqR9bX5JyRExr+0PC6MmaSe6LTNCXprrQu3ZPmtFP60/BUoIi8EymXX6fRkvandUJU0EJcxIHkrqRNIaAX4vo2H8eOAPgIgBXAEgD/hRgzHQpgmfS4sm/6eHMWpRDPlGIFekEWOAuCwAR0BoMxIkxAZzAYjPyR7Yp5CpUmgiRbPjUmgoadt5TkMQwnjEvJHnPGU1OKJE+XwlK9zx5QtlTP5+RKDc7ICeWnlH/se8C3uuncJsz484wNu87edUL04+ismitrnp9M4jkAWJdaO/wP+w9sv6N9bs2VNdv8j/qnhd4ITXWsdjSPRjyXUSaClPdFPog49py1Z1n5d8phucoCoiEIbw6X7/7W7rPMh5ibZj4+85XRjtVxZ8ec7n92T7essmwxH2Xe1vmLzhM8F3qOr32o9mWFiJ5PCtGjPApgs7SZIVZZzAawAsBxEKsw5L7prRhd4qCQCZRSSDjJ4wCF6eeXdwt3uQKdUqqaSjcGg1ESZPN7TwHRXSnP5zJZUWPcNGI8LVmq2wYTxhVtqOwQ7aCVxw0rHLaalcI4ISSo0WhChJB8tNvJlqJ8r8tPL9/b9c+uw5q/03xqw4MN/0nsS1jarm87Wd+g99hX2cddMZ5LTHNMYW2FNua7z7fA9QPXp5yFS3vv8h5EjCRtXmjuHfkIAxksZiIcAdET2vTtplWpUMow5XdT9MY5RlCBYvc3dh8T3hw+cPb62X8yzjBmUYvfT3xn3Nz6w9ZDAKDm3pr/BO4OLA7cGVgF4FX7/9kLsUihEDETRb9YTgDUo9/q/VCI7bDkvuk7IcZYox2jEDABfWzjIN9jsQp0BoORDUxAZzAYjPyRlYW7ikVkQIWJIMlSXQeASyaTszN7jqO/AmIoS3XZGnDbIBUQiRxXQOSaSX1yw1H7k9qtAOC737d6y6wtC0BAa66qec59sXvXRJ9bJvYV9gCAz/0P+w9svrB5OQA4VjuanV91tij3owLdr0/7YPcNheUwS6/zAuf2wB8DBwu7haOqzq/6bPe3dp/FGblk3c11m0d73uHN4fLAo4HDbafa2p1XOD8hHOkC8FLnLzpP8N/mP9h9q/t9FCZoL+QCkyiAj6VNC2AGxKTQQojWhVEMrLIYKZFbyB7opSagl0TSSapAZ4kgBoNRbGQtoKOIryfHiRrjJi0ACIJQlUgkFmYI43LMZM14WhriYuIQIaSX47gWpTAuLSyOTGJL9aJe6GycYYw1Pt74ZNNXmk5r+mrT1wHAMMuwb+Y/Zz43VneqfKGxafiaq2o+a7+9fX7HXR0HA4DWqY3WXF2zRW7RBYjxEYBxxU11N9e92fzD5sUt57fMb/h9w7vt69oXhd4ILXR917V+tOI5ALRe17oUPEj1HdWf6Bv0vbX31W7w/MBzTODuwErzMvM/tU5tsVagDzfWPmlbD6ASYsw0G8ApisflRcjZVPUzC/exjVUyC5xZBTqDwcgGJqAzGAxG/mAW7iNTUokgSqlWroDIFMYVVeNWSI4DyWTyK+i3VJcTPd5MYVxK+BT7hT2ltKjzQaj+UfU23/2+1QDAmbm463uupok+p6Gwr7AHep/vjaR8KSsAVJxV0ap8PNmWNLTd2LbYea5zmyS4Q4gLXOv1rYea5pr8VRdU7clmHNeVrr3ESQ7u/Fnn4T3P9Ryuq9J1Nf6j8R+Zvd2VveKV92krtCk58WQ90to9Zd2Ul3TLdXXyr4Jtja2Ds3LPGhcaR10FMh5SLSlj8N/BBuflzm3yfTRJif/X/vnOy5xbOdPoK/klhks6pAHskDYCYAr6qywOkR6Xqyx2YPAqC9YDfWzjFIq8W7in02kyiUUBBoPBGIoRf7cUv22qdO5CCcVNGZbqdkqpMnayA5AXFhsAgOf5pTzPA0BMETO1azSa7ZmtqPJgqc4YJcYZxljluZXvdvy640QAqLm65s3JJp7LaGwavuKsit2ygF5+cnlzZsziu983M9WestbdVPeJ3B6r96VeV/dT3TOn/HzKe1qndkSnAvNCc6j2gdpE2zfayI6TdlwAAK7vutbXXicu0pYR4gInJAQu8/XKjKWm/GLKmyk+NYdMEduucRaOr72vdkPsvVi5dD6Z7QhyTd+XrOu+rgOsJ1pb9TP0fbFJ6JlQDTES3rraOlbXgZHiAD+ATdJmhejoNQfAKgDHA+hEf3sszxDHKhUBvSB25xJ5d9OSKMicWAU6g8HIBiagMxgMRv7IqgIdKrZwR78VIyZzokNK8hiHE8alCohMS/WkZKkeIoQEOI5r5jguyPP8FEEQ5pjN5t8SQqIqslSfvG/yCMg9z8FBMM0x7Y1tizU0ndt0rNwTfaLPL5POP3dOT/lSVn2dPpj0JG2ynbt8rtpybYqzcQnv3d5FAN63HmHtbr2u9dBEc8LpWOPYM5qxzEf3f+wNjQaPcZZxgHgeeiNUsevsXedX/7j62Zorar4AgPiuuHnHSTu+YVtu+6zhDw2b5H0dJzp8sVisTrnYwrLcEsgYMl+vd58wHHwqOK3nbz1L+S7e5LrF9SFNUuL5nueYxNbEdNMhJp/1BKt3HONkW2XXIm2vAHCiX0z/P8XjcnV6AAA8Ho+eEIKamhomoE++cQoylpQImnS/SQwGgzECrAJ9ZIpCQKeUagRBsA7XigqADftbqodkMZwQ0in/O5VKnabVal81GAxvE0ImpQibJyb9ez0UgScCU7z3eI/TVev8QkLQ7bt835eVPdEHYcLmGt8dN3U+2HmgxqZJEA2hgb8H5mjKNZ/KPdEBwNBgCAVfDTa23dS2sO6muk+CrwarfA/6DjHONvo4G5f1Z1Jbo4V+uj6S9qQtAFD25bJ9mfvsPG3nWr6Htx7wwgH/kkX0vZfvPbzn2Z5Fc1+f+0d9nT4BAMZZxiiX5OLJZLIvr89ZON6y0pJ13/YcQNOBtC74n+AhoedCC2p+W/OCfoY+GnomVNO5rnO1Ya6hZRwCOpD99WwYwIfSpgfQCDFuWgRgufT4dgBfCDFhHzGQpMI5gFKBgiYoN44F0sPBKtDHPg7yPZZUgc4WHjMYjGFhAjqDwWDkj9H0QC/aAHmcKF+jCUn4U0qJIAgWWRgfpm+eLuOpEUXFQ4uiGqKvAoLjuMRgY8bjcb0gCLM5josM9jhjciGL53LPc/fF7l2eWz3z5J7ojY81vjqZ+qB3/rlzutzz3PlVZ0vw9aBT2ROdaAnlzJxQ/4v6D1quazms49cdhxONWNXlusj1gWOVI+vES2xrzOy5yANtlbbLcpilqffF3sV7vrMnMv3B6Zvk18RyuKXHOMu41/Nzz5kA/ll+Svm+HSft+AYf5K0VZ1VsH+X0CvJCOy9xbue7eVPo2dAiKlCS9qQdia2J6RWXVrw6TvF8rOcfAPCWtFnQ3zf9GACrAfgFQdh+1llfPTgWi+GNN17jLBZLvlbTl5qAnveqcAWF6IHOlYBjCYPBUB9MQB+ZCY8ZBUHQDyeMS+K4JeNpsqV6kBDSo7BUDxJCZLet8GALiimlSKVSpxFCIioSz4t6EVzgicAUuef5zH/OfC4dSOuavtJ0mtwTfRgRveDEd8dNcs/zmqtrtnAGTmi/vX2+3BNdFtHL1pZ5AXzc+YfOg5vOaTqOpqnGONforbup7mNOz2X1flGBwr/Or4u9H6soO7ns9dDG0NzdX999xoy/zviXUqyvOL3is5ZrW87esWbHGQe8cMC/2m5sO9T/iH912dqyt3Q1usFyC0ONn+/rdQIAWqc2VX1X9fMdP+xY235J+xr7qfaPu//SvVTfoO+o+U3NG+M9/hhIAtgmbRzEvulzAMymPD2s7YI2aphlCFTdWLWRcIRQgcJzoefYZFOyZvrL0/9G9DlfGF9K7bWAwsVNrAc6g8GYNDABncFgMPKHbOE+0kWz2i3cgTwJJQpL9ZEqIJSvv4D+vnlBjuPaBxHGc2Gprrb3vGiTQURLqMamicviOdDfEz2xN1E2sWc3CARU2fNc7okeeTdSpdyNM3NC3Q11H+3+5u7jaZpq7CvtTaMRz6OfRm0t32o5XOPWoOEvDf8wzTZF9l25LxJ4LLBy78V7+en3Td8MAJyRE2a/PPvf24/ffrrnFs/Znls84CxcZMbDMx52HO/ozOncx8eAQN11g+tj8CChF0KHAYDzMuf6sq+V7c3BGOP9LkQAfCRtOkh909evX79o9+4dBgD461//eu53v/vdtyD2Tc91wrnUBPRCLuAqiIU7WCUFg8EoPrL53WIW7nmKHyilRGGpbstcVAxAXlRsyHiqbKkeVFiqK4XxICEkPlanMcXz1BY3AUU6Z41dkzTNMzU3PNzwitahTWsd2nTj441P7r147yrOnH21dqHQVmmjrh+4vpDt0WuurvnMe7d3TuZ+ZWvLvJH3I57oh9E6AKi7vu6TbMVzAGj+dvPRoc0hTdWPq7bVfr/27egn0a1N5zadsfvru8+Y9fSsxw3TDTEAkNpoPdFybcvZnzR8crU09lszHpmxYTIt2JagAGCcbwxV31X978sFHwAAIABJREFUvOdCzxndf+w+WuvSdtU+UPsqZ+HGky/JxfW5AGCvtL1ENKTKOM94fNfvumYSMzm18qpK+H/pXxZ+IWx0fN2xKQ/iOcAq0MeK/Hee9UBnMBgTDhPQGQwGI38wC/eRGVMlicJSfUByRxCEvp55UrLHlPHUBCEkBFEg98uW6sp+4wWyVJ/wCpIJoijnTLSENv61cUPm/bKIPtmo+lbVfiKvfYU9IPc6lxHiAuf5uedgAAABDW0KTTcfZvZn7jcUxlnGiPU4q6f80nKHcZoxCgBT75j6AdERwbrY2qHclzNywvQHpr/w+ZLPDwSAsrVl74wgng9XTVEQsZMmKUm1pMrl28ntSSfEBMy4D52DY8ikAGwXBGH7Ndf8tIbjlrspdeP3v/9j/XnnnXeOXq9Pob9v+k4M3jd9tJSigF4owTnvc2IV6AwGo0gZ8bdR5WIqMMaYUbJUtw3XigrigmLlsWVLdXlRcVOmMC4tKC6UIKrW97zoKFtT5itbU/ai8j7jDGNs9vOzn52ocxoK4wxjrO6Guk+V92msGj6zLzkg9jyPfhKtBQcKAcRzq+cgZU/0kXAc52g2HmtcWnFGhRcQe6I3Ptr4L98Dvvn6Kfq4ct+qC6r2dP65szn2eWwWAEz73bSNYxDPC1KBLpNqTlnlfwtxwZD2pg3KnuhjJNfn3ll1XdVjaV/6qK67u1Z33d0FAMaq66qEiosqlkF0+FL2Tc8FTEAf+zjI91hMQGcwGNnABHQGg8HIH8zCfWT2e40kS3WrLIwPluiBmOQZzFJdFsNbFNUQykRPslATG4GircYeB2qc86RFiAuc3PPcfZn7XduRtq6W61oOk3uiZyOic0ZOqP1l7c5UKjVXeX/9rfUfZe4b3xU37zx159eJicQ0Nk24699dK41zjZ1yT/RJQl+grux57rzMuT7ZlCwPPRtaBACuW1wf5mCMnPLAAw8c4PW2uoFHAFSho2MB96tf/WrDtddeK0C0ej8F4ndwH6QegADGaqFZqL9XpViBnnexPp1OE0IIq0BnMBjFRra/W2qPmwbMXRAEw3DCuFQ1nmmpnkK/21Y3x3F75cXECsetSAEWFGfLZDmPQqG2+U56el/qdck9z+tuqvs4+ErQ1fmHzoPlnujZiOjOrzpbwuHwgN8580JzaPr9omOXkr2X7z089nlsln6Kvj3VkaqS7dzlnuiTCAoAcs9z/Ux9m/NS57ven3hPaL+kfY3cE32Mx87b73z1XdVvBZ8MHgseGmhBy79fvg7ATIgx0+EAjgYQQn/MtAdjF1mZgD72cZDvsZiAzmAwsoEJ6AwGg5EnbDYbDYVC2dizq8bCXbJUt8uJHp7nZwFALBY7g1JqlcRxKwa+HjwUFRCSpXqmMB4usoo7NSf/1MSkfY/TgbQu3Z02K3ueyz3R4zvi9myr0LMhvitulnueNz7a+Gfb0bbA9uO3ny73RM8Q0bN5zfIetPvv8s+Te55Ltu17ASD0bGiRabGp3XayrX2Mh8550kEQBDz00J+PIeRoSukx0ut3Bv3LXx477NJLL73HYrFsgvi7egDEHoCrABwPoBMDqyyyPa9SrEAvpICebwt3DiwRxGAwio9sfxsFSmnJO3dlWKrLQrie5/m5kUhkGiTHLQD6jKdGFa2n2hQV5Mqq8TFbqk8Qao2b1DjnSUl8R9whi+ecnqNyT/Tup7tnpnvSWtn+PRfsvXzv4XLP8xmPzNjg/7N/urInulJEp5RO5GeEAGJM6f+V/1h9g75Dtm2Xe6L7bvItnfLIlPXjOH7uW/xJPc/BQ6Op1ID388RzoWdZ7UO1rxGOfA7RiWMqpL7pABZB7Ku+E2LMtBNAfKjjD0IpCuiFWKgrzynfFu4sbmIwGCPCBHQGg8HIL9lc8BW9hbtkqW7KrICQLNWVFRCZluop6fl6Qkgnx3FNGdUPsqV6wedUAEpyUiOgxjlPSvR1+kTDgw0blRUTnJkTpt4x9f1srQgzGPo5FOCsXHTab6f9W7Ztn/3y7H/vWLvjFPCj/kzk8zPUl3xwXur83HSoyWddbfXJD7pucH1sWmTqsJ440J5+ohGrz9uqgV+g32H+WyQU+pd93bp1C2+55ZYPAYQBfChtegCN6E8KLZceV1ZZDFfhwgT0scN6oDMYDMbgZPvbWPRiqsJSfUArKmUFOfa3VBcAEEqpieO4ACHEmymMF9hSvaBMsFBYUCZR5X+hmPTzdV3k2gkBRBkjla0t8zqOd/jGEDcNuz9NU65sTdlmuee53BPd/7B/8Siv7gpxvU61Tm2q6qdVL5uXmLvknudyT3RimFyOSLJ4Hn4hvNx+pn19zd01x/h+5mvqvr97uedCDyQRXYAYC+0B8CIAN8SYaQ6A09DfV11ehNw7wrClJqAXahz57x+rQGcwGBMOE9AZDAYjv4xYXS4FyZM2KUAp5QRBsMjC+GCJHgB27P83JaxI7OxVVo3LNoHpdHpmMpk802QyPcFxXGICpjdRTPpEQR5Q45wnNYMlfMaQBBrxt8s4yxid/8H8R5T3cUZOmPPanCdHOVbB4EycoBTPZWxrbeMVz3OeRHnuuecWiP/61n6PvfrqawskAV1JEsA2aeMA1KO/yuIw6XFl3/TYEEOXioCed1FbQSEs3FkPdAaDUYwUvYW7tKDYMJwwLonj5oynpgghQYi26l0KS3Wl41YkEolcptFotphMptcmYHoTCYshGBMK4QjA7f85HOOi42GZfu/0tzPvq7qgao8kpE8m+n6HrausnZkPGucbQzk4fm5f3zRIqi3llMTztwAc6/qpazfv5/cltifqkAaBfr8xvdL2BsSc12xpOwHAGgAdEIX07QCGcigrFQG9IKK2BLNwZzAYkwYmoDMYDEZ+mdQW7pRSHc/ztqGE8Sws1WV7wAHCuFQBkW31PTBJE2F5ZNIm/xiMsVBAl4h8iqqFCNRzfv733HPPC6+//vpHAOB0OueUlZUd2tTU9DcAWLRokX+Ep8tVFHsBvASgCv1i+qnSue5Ff3V6D1gF+qQei+d5lghiMBjFSNYW7pgA5y5KKVEsKN6v5zhExy0bBrdUV8ZM2zJbURFCElleR6k1flDrvNU451KnkO9pQSrQ83jsnF8zEz2h056d9i/FwgcCgNbcXfMWTVOSxYKIIID3pM2A/r7pSwCskB6XY6a9EK/HS6kCvdBW8UD+LdxZ3MRgMEaECegMBoORX7IWkSmlJFeWbQpL9QHCuKKHnlwBYcx4akJRAeGTLNWVwniuLdXVKqADbM4MxnCwaqMsaWxsjDY2Nu6WblYBEFasWLF7uOcMQ6e0bYRoHyv3TT8OYqWFD0BA2reUBPRCWUwWxMJdhfavDAaj+JmwCnRKqVayVN9PGFdUjVuxv6W6vKA4JFmqDxDGpQXFuUzMq1VIBtQ1b/Y3nDEaJurzUpTfyUFEcjrE/SORALBV2jgA09C/CHmx9PhOiIuq8r3oqxQF9EJZuHNgra8YDMYIMAGdwWAw8ks21eUDVsCOdEDJUt06VAWEom/eYJbqQWnbq6iGCBFCghqNJkgISY16huNDlQK6SsUNtc1ZbfPNFdn8FhRzBXq+j59L0TkE4ANp06O/ymKu9PjFGNg3Pder9wtVEVDoCvS8W7iDVVIwGIziI+c90KUFxcbhhHFpYXGmpXpSWlAcIoQEOI5rzhDG5QXFhb7WU6uAzq6pGaVELj/PBBP7m1AsMc1wY+QCAUCztL0AoBr9fdNNAJYCqEV/3/RgjsaVKUi1dgHHUY7FLNwZDMaEwwR0BoPByC/ZJDrkC1AiWaoPKYxL4rgl45g8xIpxueqhTSmMS4mecJaW6oVGlQI61JsAU+OcJz3Rz6JW40xjlDNyfb8RKX9Kxwd5rXGGcaj+1xNFKXyG8p1sygdJAJ9LWxNEe/dtECvUF0mP74SYFNoJIJ6DMUvVwj2vf4uZhTuDwShSRmXhnmGpPmgrKoj9anUZz49IFeNBjuNa5H8rYqYQx3GJnM4sR0iCfSlcB40WVc6bUqq6OU92qEAR3RK1Ww62DBBAIx9G7JZDLbkWRcdLvgXIQnw+i9XtqkPaXgdwBQAPAA2AEwGshdgrXV6E7M3BeKVYgV4oC3e28JjBYIwIE9AZDAYjv/BAXwWEWVkBoUj01AJAJBK5AvtbqscVFRBejuN2ZgjjQUJIrID9h3NNn339RJ/IBKC2ObPqkUmIEBW4vRfvPVFfq+9q+FPD/zgjJ6T8KV3zec3HA6Cznp71POGG/ahOus9x10NdMw1zDd2WZRbZahyRDZGq5J6krfy88uGszQuSaKI8Re/fe6frZ+p7zUvM3fKD4fVhlxAU9PbT7a3jHKNQ37WXIVZZuNFfZXEa+vuqy1UWvWM8fikK6ByAdD4HSKfTXI4tgxkMBqMQ9CXJFZbq+wnjAEzpdPqodDq9EgOvQZSW6kGO4zoGEcZzbaleaFQpJEuoad4sZpqkdNzRMbf76e6FdTfXvepY7egEAO+93pn+v/iX1P2s7uWyNWW+EQ4xqT7HqZaUseexnlmVV1V+Ksd7Qlzg/Hf4FzgvcW7VlGlGumbN+6Lg+Na4LfZOrMrxVcceebF3ujOtDz4ZnGo/xb5P69YmxzlOIb5vrQDegJjrmwkxZjoSwEqIcZIcM+3F2ATjUhTQC2LhLuUhi/m6gMFgFAAmoDMYDEYeSafTrkQicQal1Axx1akMhWipHoJ0wabRaN7mOK5b7jmu0WhCE2CpXmjUXIHOYEw4nJkT3Be5N7evaz+2+fzmlfXr6t/a84M9q9L+tL1+Xf1LI4jnwxL7ImZJ7kuaHceLCSYA4MO8pvvp7trKr1W2jPGww4qdNElJ+MXw3N7Heu2un7tetCyzBCIbIlW+m3wnaN3arrJzy5rH0OMuVxDpv5RoiRB+OTwNAMxLzN3h9WFX9K1orWmxqSMH4xQ6eeKVtjcgVvvJfdNPALAGYgWGXGUxmvkVStjOe19yBXmfUzqdZokgBoNRjNBYLHYEz/MnQbS8VSJbqgcB8IQQr1ar/VR23JpAS/VCo1YBXa3zVuOcJzVV36vaGX4vXN92Y9sqAK/Gt8cd/r/4j7QutW53nOAYSTwfEipQBB4NTHN+zblPGad0PdlVa1tm8+tcurGIxCN+fsKvhmtCz4QO5bt4k/s297s0Sbn2S9pXJL9ITjEtNnXYTrQNVx2d788nAUCJnghpb9rS83DPjLJvlu0WQoK259GemYQjFNqcXFMXsso9DuAzadMAmI7+9lhHSI/vhBgzNUHso57tGED+51IQUVuCWbgzGIxJAxPQGQwGI49IQvheAK2yMC4levos1ZPJ5IHJZPJMg8Hw9mS1DMwjqhbQKaUoYveAsaCqyRYLFWdWtAF4zXOr57gdX95xNtGR9NQ7p75gO9rWpdyPD/MajVUzIMAU4gKhHB00sPX8wrM4+km0of62+ufL1pZ5+TCvafp60/GJXYla6xHWvxtnGqPDnNaYgmWiJ7T2gdqXPN/xnOi73nei/Qz7+8F/BhdrXdrumvtr1o8gnhckUCccofaz7PuC/wgi/HJ4Wuy9mIvv4k2mxaYO21rbeAX0ie4XGATwvrQZ0N83fQmAFdLjspi+F8MnLEqxAr0QFu5cvsdgMBiMPCBwHNdJCNmpaEslV4/3iUfhcPhyjuP2GgyG9yfyZCcItQrJpb4wIhO1zbdo0Dq06YY/NrzWfEHzsa3Xtp4IANal1u1T75r6rnLRMRUohKiwX9xEkxSDib7hTeGK9jvajwu+HtzR8GDDJqIl1P+If2r7ne2rHCc4tky9Y+oH4zjtIT9P5d8qbxbCgr73771LvFd5Ob6XNyW/SE6puLTifyOI50Bhrp+pYZYhYj/Dvjv4r+CMrt91zQEPjugJ7zjXsUvr1I6n2KRQv6VDvU48RJG8CcDzAGogLkCeDWCB9Pge9Fenh0YYA0OMk0smwsI93xXozMKdwWCMCDfyLgwGg8EYKxqNJmg0Gj8wGo3v6PX6bTqdrk2j0QQz+pHLF4Vq/E1Wq4Auo6Z5s2TQJMZ2jK2vaoIzcknL4ZYe5eOx7THLFyu/+KrvAV+jfF+6O63dd96+hT1/GrBrH9N+M+1Nfa0+0HJNy9ru/3TXyOJ57U9rXxpBPB+UZFPSLMSFAUkIPshrk01Js3I/bZU2Wftg7Ys0Tg29f+1dSnnK1TxQ8/I4kyy5oO/7TjgC+1n2faAA38WbOBuXzIF4PmCMPJJtQiMBYCuAJwHcDuARiEmgAwCcC+BKAKcDmA9RbB9snEIlaAolOOe92p1VoDMYjCKFGgyG3UajcaPBYPhEr9fv1mq1fqV4Lu8HdV0/K1Hz3NU6b8YkQ+vQpq1HWPfJt8tOKmvOdOzad9m+I5rOblrDh/g+B0Lvvd6ZbRe0aVMdKV3mMW1H27pc33e9Hnk3ckDzd5qX+R/2T2u/s32VaZ5p35SfTflohFPa77qSChSJbQlL5v3xLXF75n3Oi53bHWc63o1ujs5OfJ6YWnFpxf8cZzrG6hSWS/peVMMsQ8SyytIqRAS9EBe09rPsu3MQ1xVSdM5mjHYAGwA8AOBuAOshxg1rAfwIwIUAjgbgGmIMZDnOeJgIC/d890BncRODwRgRNYo1DAaDUUgEjBzwq1ZEVlgtqm3uap03YxIi9zznjFyy/JTyD4WoYGw+v3mlEBf6rhMNDYaofrre235n+wm+B3yN6e60dtdZu76U3J20mRZlOq2KaJ3aVONjjS/oXLqelmtbTo5/Ea+v/WntS86zncP1+B70O0HTlHh/6j3af5u/RoiLcTQf5LUdP+5Y5rvBt5ym6YDnJT5LOPpuCOAS2xL7JYyGGTuvPdDl40dejfQlQISwoI++HS3P0RiTMXkiAGiG2DP9NwB+D2AzgEqIIvpVEEX1xRBt4OVxSrECPa9j8TzPeqAzGIxiJFtxWM0islrnrtZ5q3HOkx7vvd6ZXU90LbYssjTpp+u9bTe1repd31ul3Md2jK0l0ZyoafqKKKJ77/XO9P7Wu8owzwBt1eDCr/si9y5ZRG+/o/0404Gmlhl/nrFB7vs9BIN+Rnof7W3wXu89Kv55HJCuO3sf753qvda7OvR8qFq5rxAXuPjWeN99sfdiNVTI6lI139e0fcdPd6b10U3RGvmB0DOhemWcOo7jA5NHQFfSA+AdiIuPb4e4GLkHwFIA3wdwGcRWWdMh6jqFFtALsfCYVaAzGIxJAxPQGQwGI79kI6DLF6BqDJLVKiSrtRpbbe/zpEeIClzzec3Hyz3P626s+7Tm6prX4jvidc3fbl4hJ1A4PUcb/9a43nyweVf7He0nbl209bvJlqSr9o7aT40LjUMenxiIQPQkLd/W2DRjqhYgWkIrvlvxfmpfSh/4dcCe8qQMHT/uWJbuSJeXf6/8faU1u9zzXFev89Y9WveE1q3t8l3vOzGyKeIcy9h5gCp7nlf9tOpjwwGGrvDL4Wk5EtGL4felA8DrEIX03wB4Sbr/RAA/BPAdAGUYvDI915SUhbtUScEs3BkMRrGR7e+WAPXmsdQqJBfDdU0uUdt8gSL5XPse8jXIPc+n3TftrYY/Nrymq9V1td3Ytiq0KVQh71dxWoWn7ud1zyWaEzVbD9t6ofce73GWRZadlVdV8kQz9FQ1Nk2f4wZn5FIjtJ4aEvtZ9r36Rn2g+4FuRDZGynsf753a82jPYuMhxj3WE619jldCXOjree683Pma4xzH29E3o3O913gPz1JEzzc03ZnWyz3PKy6q2Oo4x7GT7+JNPQ/3zMiBiA4U5vs2njHiAD4F8E+IYvpjAHYBmAfgmwCuAHCktG++2/SWnIW7IAis9RWDwRgRtQYeDAaDUSiyrkCXVj+qDXnuRRE05wE1zXtSROGMgXBmTrAead1Tv67+JbnnecWZFW01V9e8Zltm26e0JOT0HJ1+3/T/ybdty22fG+cbI5QO7IHOh3hNujetlXueJ/clXTVX1rxsmG7oaLmmZW3P8z3uoc4nuiVqy7wvviVup2lKLCssfuePnB2JHQlt23ltX061pKoqr6p8w7LcEpD3pUlK/Lf7V2hd2u6aB2peNswyRGofrH1RU64J+df5j86sVM+gIBXolKdI7EiUyT3PZTt3wwGGrsS2xHgF9EJVoOdyjF4A7wJ4FMAdAP4NoAtiJfpsAJdDFNYbkJ/YpaQq0JmFO4PBKFKyEocl9yo1XT8roSxmYjAmDusSq9+x1rFF7nku90S3LrXuNM0zBZX7VpxW4bEcYdku3556z9Q3hIgAZFwHJj1JAwDIPc9NC0x7qi6s2hh5PzKz+TvNy4aKXRJ7EiY+yA+4LhbiApf4ImHlTJzgusX1iX62HoG7Agf1PCKK565bXB8oY7vuh7pnyT3PHWc6WpwXO7fLInroqVDdCC9HISrQkWxKWghHqNzzXO6JLgQFQ9qTHnoVd5bHLwC5fJ14iOL5cwDuAvAQgPcByPHjJQC+CuAwANYcjalkIgT0vIrb0t9UFjcxGIxhyffqJAaDwVA72VzwqbUKG1Dv3NU6b7XNtyio/Unt1sz7Ks6saMu8L92d1jZ9rWkt0RKeM3Ox4IbgQckfJJO2M2zEeo4VhCPgQ7ym9brWRURH+HRX2iz3PHee7WwtP6W8relrTWtarmlZazzA+ERmH/Se53vcrT9tPani0gqf/XTRxTuyIVLlu9l3vO1Lto8rr6jcqm/Up+TED9EQwXiwcUADdqIn1HWra71uqi4q98aTe6Kn29PGsVZy5AgCAERDaMWFFTuJvv9c+nqip8f9HSlGAV1JHMBn0naBdLsbwFwAR0i3d0Lspd4Esc/6eMl7X/JCjsXzfFFZERJCygEsBBCF+Nrw0ib/W1DcRwAEKaW+iTlbBoORRyiy+31UtYAOdc5drfNW45wnNeYF5pB5gfkT5X1ahzY99Y6pH2Tu673XOzP8VniuxqEJ82HetPOknWeajjVpKs+v1Btnirpv93+6awJ/D8x3nOzY3nFnxyrTPNM+2badaIngu9+3ovX61kj9bfUfKo9NBYq9l+1dSWzE6brd9anGrumrJk970uX1/6p/ijNx1HSICZFXI+K5LzW3ZfZqr/h+xQ7ToaZO81Jzl3yf82LnduNCo9+81BzAxEIAUPMSc7fpUFOPMm4yzDJE9JfpP1feN8bjA8UbN1EAHmlrB3AWgP8BmAngJABfAtAGYDvEuKkzB2PKCzYK2QO9EBXoLG5iMBjDwgR0BoPByC/Mwn141Coks2psRlHR1/O8Jemqv63+acdJjo6mrzatjn4SXUi1FNiDee5L3V+0Xte6KNWRslf/qPo9oiM0sTvRJIvxck/0wN8D0zLFcwBwnOjw9rzYs6vr910H8738Pn2Dvt13s+94bbW2q/zb5dv5IK8N/CrgJjoi2E62bQm/FD7Q+2Pvcvev3Bs1Nk1f4Gs6xNQLiMklvpvXaZ3alLZKm9RWaZMAkA6kdbK4nkHBVtUPlvAhHAH04x67EL+lhexNHgTwvLTVAJgDsSp9AcTEwB6ISaHtAELjGKdkLNx5ni+2SoplAJ4GkAKgG2Y/HoAGwH8BnFKA82IwGIVFQL9QOtxvMrNwVx9qi5vUNt+SQ+55bllk2Tnj4Rkbuv/bXdN6XeuXopuixNfjm6a9TPth/PO43f+Yf4H5YLPHeY5zH9/Fv+n6rmuX3PPcfZF7F2fhUvaV9v3EL8IRVP+o+t22n7ed3nFFx3zXTa4vfDf6jkruSNY5L3du4EycEHwqWBN+OQzjIUYv38lHuu7rOpJw5C3rCVZv33G0hMriuTI+kt29+C5ex5VxqUzhXX46CvRZHTRuGp94DhS/gJ45BgC8B2ATABOAAyDGTMsBHAtxQbIcM+0b4zmVnIV7EfZAZ3ETgzEBMAGdwWAw8suIArpkRQioMxmkdgFdTfNmyaAiRggLWvDg6n9Z/0z5qeXtAND4t8b1zRc1uzX1mobIe5H63efuric6wlf/uPo96xJrDwBYDrP0Ko+jdWpT7ovduwYbg3AE9b+uf3/fjftqu//cvRg8eN1Unbf2gdr1nJXjPd/3rIAAQ+VVlUHDbMN24wJjwH+nf7n3Cu+ymt/VvJFZXR64K3Bg4rNEjeuXrk26al0CAHr+1jMt9FRonvuX7v/pG/X7ifh5ZlyJgOib0QqujEsZ5xn7hOJUa8qY2JpwKJJhxV6BPtw47dK2AWJ/9NnSthZipYUH/VUWo1lpX8ie4Xl/7YqtAh3A2wBOg+guoIV4LaTciOL/ZojOAwwGo/RQXhsP9zupVhEZUO/c1TpvNc65JEj702ZZPCdaQitOq/AAeNb/tP/bRE+E1p+0LgMA8yHmttqf1H5GOILqy6p3ZB6n6ltVe4caw77CHuAN/BbP1Z6FLWe3nA0AlT+qfNV+qr0t+J9gXc8jPfPtp9nh+Jrjc5qgHu/V3iMD9waOIkay0bLC4lceK9mUNHuv866wn2Lf6viKYx8ApDwpg+8G31HG+cYO5+XObTl9gbJjzNfMQlzgQs+Gau2n2VuV4n/w38Ep1rVWD2filDm6QonbhRhDnksMwCfSpoXYBmsOxAXIR0qP70C/o9dgC8uzGSefMAv3wWFxE4MxATABncFgMPJLNhdjau4DrkYhuQ9KKSFEVVNX1WRL6Tutr9fHZ78y+8nMnui1d9d+kYqlpnkvEPVbTZkmZjnc0jPUcUaCcITYTrD5Q8+HLABgOsK0R1OmSQOA+UjzPtMSk9sw22ABACn5szHZnLQNZs1uO8m2L/ZBbKrvWt8y1y9dmyKvRaqDTwQPMR1u2q1r0A0mnhekB/pYjk8Fip5Hehbw3byl6saqjcZ5xlCqNWX0Xus9GgKIeYV5vVyxMpbjj5JCCejD2Z33AHhH2owAZkFMDC0FcIz0uLLKYrjkSyEr0Ath4V5UiSBKaSeApyb6PBgMxoQjV6CPhFrFVEDdc1frvBlFSN1NdVuoQKGMmypOq/CCf3TxAAAgAElEQVToVuv42PpYV+9j4vriijMqmoeo7s4K4zxjlLNwKT4oXvZZVlh8AGA81Nhl8ViaHV93zCKEgJiI4F7n3hy4OzDPuMDYm3kcXYMuajzI2Nb7j96FAGA+2uz13eA7iiaozrrWum+I4QvRA31Mxw+/HK7ufbR3Ubo1bau4tGIb4QgCvw3MjrwSmQuC9+yn2ZWtykohbhouxkxDbH+1U9qvFv2OXgulx3dDjJm2A4hkMU4hFh4XxMK92CrQWdzEYEwMTEBnMBiM/MIs3IdHrQK6Gqux1TjnkiIzwZP0JA0tP26Zr5+jJ0RPeNNBpvbI+5Ep247edpbre67NlV+vbBntGMH/BZ3en3ln66fpWzkH1xX8Z/AIzsSlK75XsbP8/PLdAA6CuJoagCiiZ1ZRyBjmGMKuG1ybfLf4lrV/t30NAJgON+2uvLZyy3iSVRMB4Qiqbq5623uNd3nnzZ3LK75f8V73n7oPoSmqcd3iekMhnhcySZNvsp1LHMCn0qaBWGUxG8A8AEsgVlko+6YnxzhOLiiEhXtRJYIGgxCigfg9rwDAU0pbJ/iUGAxG/pF7oI8YN0kJbzWiVgGdUqqqMEJVk0WJzjcz1vD8wjMvEUsQvp13GWcZP0sH0uZ9P9p3DDGQ9MwnZr7E6blRvQ5CXODar2mfK4QEvW2N7f3Ia5EDPT/wHFd7X+0r+mn6mPMi54DKU87ECVXXVH061LlWXlO5xX+bH71P9B7c+0QvOCsXr/pp1ZuGAwzDCaqTEvuX7Z60J701/EJ4HgBAAyHySmSu5XjL5wrxfLLFM+MdA1mMQyH2RW8D8CrE6+zZEAX1LwE4GUAr+hchZ8bYzMJ9EsLiJgYj/6g18GAwGIxCkU2iQ7UW7gr7elUlg9Q6b0ZpkWpPGWLvxyp7Huvhyk8p31Z7Te1WPsSb4l/EG7uf6p4z2uOF3ghVtP2kbaWuXhet+V3Nxprf1mw0LTFt73mkZ2n3n7pnSLuN6jtjmGMImw4x9Qn5ZReUbR9GPB9zoM738FrfDb5D+RCvke+jAkXnrZ0HJXYkLOM9PgDoqnUJ923ujVSgnP9X/uV8N2913eJ6IyOxVWo90Ec7Dg9gF4DnANwF4CEA7wOoBnAWgKsAfBXAYQCs4xhnrBTCwp1ArCYpSgghVQBuBtAsbY9K9y8hhNxPCDlwIs+PwWDkjWyvjdUqIgPqnbta563GOZcskQ8jdT1/6dEIEUE75dYpH9qOse2JfhydHfs01hDdErWN5lhUoNjznT1Hxz+LlzuvdH5W9ZOqLVU3Va3n/bzD8wPPcUJMkC2cgSyvOwlHUPatsu3ybeOhxpYRxPMxX9P2PNzTEHox5FbeF307Wh64NzA7F8cHgIrvVey0rrFujWyIzJbFc+dFTqVNfin2QB/tOF0ANgP4M4A7IfbWDgNYCeAiABcDWA2gXhqj5CzcBUEoagGdxU0MRmFgFegMBoORX7K2cIc6g2Q1zx1Q37zVNt+SxnyIOVhxYUVT7397F3hu9ZzQ+YfO7sSuxNSyU8vesK+0D2X3NySmeaaQdZl1d8WVFQKxkDQAVN9Zvbnz5s64aZFp0Crzkej5W8+06KboLN10nS/tS9tlO3e5J/oQjDopEP8s7oi+FT0geUmyoua3NRs4C8f7fuJbHN0cna2r04UMBxiax3P8PtIgymfT1H5VeKXcA320UIh90T0AXgNQjv4qi5MgVlq0QaxaLxkLd0EQisrCXQkhxArglwDOB3APgJkQqykAMclXC+BEAJ9PyAkyGIx8wizcR0bNc1fTvEuyIlvt1FxT807779uXRN+IOneesnNtfHt8qrHRuM+63LrT2GgcrLXUkBCOwHqUtcX+FbtgPNrYDQCWZZYAbsL62NuxaqnH96iQe55zFi6uqdSEoxujM3treiNyT/QhGFNbquhb0anJ3clqAP+znWjzRt+Olnfe3LlK49QEhW8LO3PVlmpAnJSCJsNWv5Bi8GQV0JVEAXwsbVoAMyDGTAsBHCU9Ln8WClH4U5DiomKuQGdxE4NROJiAzmAwGPmFWbgPj1r7v6suMUIIUZv9YslDOIKqS6tarada5+49dq+N7+Vt5aeVvzXt7mnvjuV4Wqc2VX97/UexWGyB/GEhHIHrZtdHymGR5fen52890+Se55XXVm5J7khafbf4lg0joo/5d8iyzBKgV9PX/Ov8x7Zf0n6M1qkNx96PzXR83fF22TfLZPF8XL9zcs9zoiN85Y8q/9f1UNehnTd3Lpd7oit2ZQL64HQDeFvaTAAOgCioz4aYpLkEol3hFwBacjy2TCEq0DkUpjdhziCEcJRSAcAqAMcDOI9S+jAhRHYMAIB2iH0ZD5ig02QwGPklawt3qNC1S0KtAjoLIBhFj3Wxtbd6TrXg+aanO/phtBEAZv5n5lO6Sl1qLMdzfc/VlEgkzKlUqu/7YVlmCViWWQLSzaxF1VSHKJ7TBNVV3VD1pn6mPuK/zX9Q7z96F4KAOs52DNaWa0y/RYQjqP519aaOH3YsD/wqsDKxLbEl8nJknsapCVbfXb0hoy3VmJF7nluOt3xONITKdu5yT3QF+e7jnu8x8jFOGsAOaSMApqC/PRYAfAOi45fcN31UC0CypCDxpiSgs7iJwWAMi1oDDwaDwSgUIwroCjtvNf4mq7UCXa3zVhOqSPZRnqL7oe6+367ga8H5kY8j9jEfL4eLaTRlmoTpSFOT3PNc7omudWuDnJkbbqX5mN4762qrr/LqytdSzanq2PuxmfYz7O9VXFixS7HLmJMbVKDw3exbIvc8Ny81d7lvc28kZpL03+o/SogL8ntQKHG7GAV0JTEAnwD4B4BtADoA7AGwAMB5AK4EcArEygtdDsctRA/0YqxAl78bDRArJl6QbmsBpAkhBGISSADgKPzpMRiMAiBXoDML96FR69xVNW/xT574z4k8D0buSWxNILkr2WfX3nFnx4KJPB8ZzsjxWpc2JPc8l3uim44wNWvKNMlhnjqm63TOwvHVd1VvJAaSCj0TOkRICPrqu6s3aJ1a5WKCMccBwaeDtUrbdqWde+g/oSmK4xeKYhPQlVCIC4tfgRg3AWJ7LDOAL0OMmc6HWKXuzOG4eY+ZgKKtQGdxE4NRYFgFOoPBYOSXrC3cVViFDTAhWW3zVtt8SxqapsTzQ8+c8KYw577E/ZJjjaO16WtNZ+w+d/cZMx6d8S/LwZZgLsdLeVIGrVsLoulbdAQhJnBCWNBqq7T7JXdsa20dtrW2DuV9hjmGsPs293tDDDGuzycVKCKvRqbJt2MfxOr5EL9LY9Nk/h0YdXKDcAQV36n4mHNwKbkfodwTPbktac9VtUa2p4PiF9AziQJ4RhqzFqJwPhuibWEawG70V1kM1w9yJPJu4c7zPEcpHS7ZOZmRvysm6f8aAAKllBJC7ABsEBc7MBiM0iPb30ZVialKpEXXqpw71DVvVSzCVRvdz3RXd/yiQ6ObrgvP+dOcv7Td0LYo8HhgKQDU31b/YS7HogIF7+P12uqBKf9Ua8qom6KLZ+6vKdOk3be631feRziCqqurPhtmmHF9J+Ofxu00QfsWqcbei1XYTrR5M3Yb03fBtsbWDoJ37V+2e+T7Kr5XsVNXpwtbVlvkMQpRHV7oCvR8I4/zEYAXAVjQ7+h1DMR+6X70O3q1Yexzz3vMBBStgC7D4iYGo0CosdqRwWAwCkm2VoTIYr9SRK0CuhoTI2qcc0kTfjtcHtkUcbludPE1V9VsMy8whxofa/wXCGjnHzoPzOVYQlzgvFd6V3bd11VJefGjJMQEruPKjqO8P/EupUJOP17DHiy5J2nyXutdzId4jXwfH+G5vSftPT2yMXKg4+uOt6uur3o53ZqubL+k/RjFfuP6nTMtNvXI4rmMrlqXsBxj6VTcxXqgj28cCjHZ8yqA+wD8FmL/dCPEnulXALgAwFIAleMcKy8UaQ90+TXZDvG1PkW6rUf/NdJiAFMBDLUAhsFgFDfMwn1k1CqgqzWGUON7XbJ0/rHzEP0sPa3/Q/2nukpdatq90zaXrS17v+e5noOTnqQhl2P1PNwzo+OKjiWp1hQgfX9Cz4Rq2i9qPyHyemQs169DMex3kwoU/l/7DwyvD7uU9/vv8c/u+GHHKVq3tnvKI1OeNMw0eAK/CqwMvRhyK3Yb8+ef6AlViucytpNt7YMsOi4VAb1QMRMUY0UgiumPA7hd+n8rgEMgxks/BnAyRJF9tAWchZwTi5sYDMawsAp0BoPByC88srMiBNSZDFK7gK62eattviWNbZmtq+GFhreJkxwt32deYA4d8MwBf9fX6/erbhgPnJET7KfZt4aeDc3t/n23wXGuQ+O91ntkam+qynmZ882MXnZjJauDpFpS5vin8XrvFV6z+073JqIl1HO+54RkU3K640zH/xS27a/51/mP7flTzyznZc4vULgkSiklaSZ6nC4Am6XNjP4qixUAjgMQQH+VReswx1GOlW8Ld45Sms7nGLlG6uMHABsgWkT+mhAiv95mQsiFAK6C2G/x7xNzlgwGI89kZeGu8ipstc5drfNmlBCNjza+HIlFFmkd2jQgVnhPu3fa5mRL8iN9rT4xxsMO+r2wfdnWEv8wPi/w2wDKvllmTTWlaroe6jrCMMfgMR9pDgz2nFyNPYA0SGpPqjyyITIbwJvW1VZf79O9tT0P9RzH2bmw+y73Bl2VLlV9V/XGjh92LO/+ffcRlpWWZyWRuxTaUhUq9itItTaGn08K/W5dBEA9xJhpDoBDpcebIMZMOzFy3/RC9kAvKgGdxU0MRuFhAjqDwWDklxF7oEO9Yqqy/7vq5g6o1rafUULo3LpUOj1QqzNMM+RCPN8vYLafbm/VVmsD3X/odoeeCZ0KDQTn5c5N1uOsvhyMN+zYSizLLQHweMN/p//ojss7VoCDIIQFc9XVVf8pv6B8t7yfdbXVp3VpnzcsMGRa2ec7UcME9NGRbdIpCuBjadMCmAExKbQQYt+/KPoTR7shJorGOtaYKdIe6AAASmmKEHIjxPf+FvQvNvg9gKcB/IhSOh4LfQaDMXnJNiZQs5hKocIF11JPcDW952qtuC9pNDYNz5GBX1/CkVzFTQPQOrUp189cn3Q/2H2o76e+xTRB44YDDa3un7vfI3qSy8/XsMciekLd69ybvVd7jwzcE1gafSvaHHsvNsO83PxJ1U+r3tVV6VJAf0/01L6USVEhPuLxx0mh2l4VgkLGTMhiLApgn7Sth+jaJbfHOkXxuBw3dQ0xFuuBPgwsbmIwCgcT0BkMBiO/ZHPRxyzc1Td3Nc6bJYNKk4J+hk2LTbHuP3SLAxtJ0rJygH35eMl6LpaVFj9N0Df9v/YfDQAVF1W8OZhVoHGhUSmel0p/8lIT0MdSFZ4GsEPaCIAp6K+yOER6vAn9iSG5yqIQFu5FmwgCAEppAMBFUkJoEUQ7ws8opbuHfyaDwShyZAv3kWAW7upDlfNmC60ZI0EpHfJ3U1OmSZuPNiP6jngJ6jjTsSPH4nlWn0/OxAnude7NLae3nBp7J9ZIDCRV85uaNzPPhbNwvGGuITza448DZuE+tnEwhrH8ADZJmxVipfQcAKsAHA+gE/2OXh70/+azCvQRYHETg1EYmIDOYDAY+WXECnRFFbYak0FqFJIB9YrJanufGaNnyM+IEBO4wN2BSqIj1HS4qSn2fmyG9yfexe5b3e8RbU4SQlknBYSYwAWfDh4g3w4/F55lOcbi1dg0wwXghUzU5JNSq6YYr606BdAiba8AcKJfTP+ytE8LxKRQ3qspeJ4nxWbhPgS9EHvRy7bODAajtMnKwh2iZqTW60m1zl1tfwPUNl81kevv75DHC78Srgw9E4J+pr6L7+STnes6l1VdV7XRdIipN4fjZ/VZDa8PV/c9IUW1kdcjVdbVIzqI5TsOYAL62MbBOMcKA/hQ2vQAGiHGTYcBWCY9vh2iXpXvRceAGJsVrYCugMVNDEYeUaNYw2AwGIUkm2oKCqh2lbma5w6oS1BmF/KlS97fWyEucB1XdhyV3JU0VlxS0eO62fVR+fnl7yS2Juq913kXU6FwHy8hJnAdP+5YmtqbclVeUfl61bVVG1JtKaf3Cu8yPsRrCnYiQ1NKSZpiHCcA4C0AfwJwJ4D/QqxAPwaAAaLd+3EQq9Zz/jdA+ntatIkgQoiWEPJlAP8A8AHEBNvjhJATJvbMGAxGnmEW7iOj5rmrdd5qQU3vb96vbUPPhmq6H+yebzzICPfP3R+773Bv1JRpIp2/6Fwe/yRuz9EwWb1nwf8Ga7sf7F5iONDQOuWJKU/pZ+i9gXsCS8Prw64cncd4YQL66MYBcrcYOAlgG4CnIMZMfwHwGcQ2WQsBmAGcBeAgAKYcjdlHOp2W58PiJgaDMSxMQGcwGIz8wizch0ftFehqmzeDMSaIngj6afpu56VOv3GBMQmIPdHLzy9/Rz9N30O4nHyVskpyhF8OV6f2pqqclzs3WldbfZaVFn/lFZVvpNpSzvCL4dpsj0+T+y8cGuy+UcIs3CfXOBEAHwF4HMDtEK3deyDavF8A4McAToZoZZgTZzCe5zlBEIoyEUQI4SAm0J4CcATEaooIxMUHLxBCfj6Bp8dgMPJLtlVTzMJdfah13mqcM2N0DPqbqZuqi5gON7WXfasMREeo1qlNude5NxlmGzo0bk0i3+P3PZikJPjP4HzDHEOb+1b3uxqbhnevc2/WN+h9vY/3zh9hAfSA6/M8xE2sAn1s4yBPYwkA9gJ4CcA9AN4DkABgB3AqgCsBfBPAEgBluRgwlUrJ1xJ577WeD1jcxGAUDmbhzmAwGPklm4tLZuGuvgSBWqux1fY+q4GCvKeEI6i8snIrgHkQ7d4AiCJ6IcZXYv8/u8d4sPEF/TR9TL7PstLi1zXoBtw3CH1Jh5QnZfBe5V1hP8v+mdw7XYgLnPcq75G6Bl1X5Q8rt43x9JiAPnnHSUn/3wIxKVSPfqv3Q6XHmyBave9Ef9/0USEIQtFVoBNCOEqpAOAcAOcBWEcpvVbxuBbAbyH2+HuXUvrfCTpVBoORP2TXLlaBPjRqnbva4ia1zZcxdgb9PTAeZAwaDzLuBHC0fJ/WqU25f+F+P99jD9hBT6h7nft1baU2Kfc8l3uiC0FBO8IC6L7r88A9gTnJXUmne517M2fiBAAIPR+q7n28d4H7NvcbulrdWBYFFLLtVSnFZijQWDGIAvofANjQ3zf9OAAnAPBBjJm2Q+ybPmpSqVRRVqCzuInBKDxMQGcwGIz8MuLFJSGEVaCrc+6AuubNkkGM0TARn5eskwKDCeUjiOcD0FZqk5pKTbj7/u6jALxlPd7a4b3Ke2SyKVltXWNtyv6U96NQvymlJGxzBRoH6J8TBbBP2tYDqISYFJoN4BTF49shJoe6sx2A53muCHugy73hDwfwKYC7gL7KClBK04SQXwJYCrF6nyWCGIzSI9uYQMhin1JFrQI6oN55M0qLfHyOs2oXmCdGPPZg4jZn4gTOxCWzHcQw29AdXh+e673ae6R7nXtzZEPE1XV/15H6mfoOTYUmNfIRBqWQYnCpCOhywU+h4kA5TxqCaE/+AcSF9DMhxkyHQ1wkEkJ/zLQHWQriRVyBzuImBqPAMAGdwWAw8ks2F2OqFZEJIaqcu1rnDfXNlzEKKKUQBMFMKbWMtGtBTkgi1ZIyEj0RtG5tX6KHJilJ7EhYjfONoVEcqi9RQ/SEVt9evbnjqo4ju37XtbTrvi4QDREqLq5407bG1jGO02UV6GMbp1CJk6HEej+ATdJmRX+VxSoAxwPoxMAqiyFfF0EQSBEK6PJ8UhBfixgASNUVMkEAXeiv5GcwGKVFthbuFOp07QLUK6Crdd5qnDMjSyilOkqpFaJ4OBEMep0e3xK3Gw8yBjPvM8w3BEfZcqvv+NYTrF4AbwXuDRzVcnrLqQCgn633uH/pfoczcmO9hmcW7mMbBwUaa6iYKQngc2njAExF/yLkRdLjOyHGTDsBxIcagOd5AkCglBZboQeLmxiMAsMEdAaDwcgvcqXZSPsA6kwGqVVILraLdAZjOLL6PAuCoBcEwS4IgoNS6pD/L212AA4AGmn3gwF8BjE4bkVhfisGTQq0ndd2Ck1QXf2/6p/QurVJmqSk9WutaxPbE9Mb3mj4vaZMM1qxkgKiraHrZ653Ws5oOQ0UMB1mah6neD7g+HmkFAX0Qlagj5ToCwP4UNr0ABrRnxRaLj2urLIY8PmTKtCLyopQcb6vAJgPYAWAZwkhBACREkLHQZzruxNzlgwGI88wC/eRUevc1RY3qW2+amPE95dSSgRBsAqCkBkv9W0ATIrjHQRgK8RrQ1kwLITYOeDYPQ/3NHTe2nmK8wrnMxUXVuwCgOCTwTrv9d4zyr9V/mrlVZWfjfX41hOs3ui70d2xzbGZAOC60fXeOMTzQcfIA0xAH99YI40jQIyF9gB4EYAb/e2xTkN/X3U5bupVPjmVShGFG2jRwOImBqPwMAGdwWAw8ks2F2TMwl19c6eAGBxP9IkUEDUmg9Tw/hIAoJRywyR67BmJHpkQIaSXENLLcZyHENILIJVOp2shCunzASyBuIL6c4iJoYLj/JHzfx0/7vhKyxktZ0/5+5R/dvywY1Xsg9g856XOZ0Ypng/4PAhxgfNd51sCAhAt4WPvxWYE/xvskHuij5FSq0AvRFKjkAI6RjlWEsA2aeMg9k2XqywOkx5X9k2PFVsPdELIAohzigHogZjceoAQ8icAb0r7rADwDQBPAnhngk6VwWDkF7kCnVm4D42aBXQ1zluNc1YFgiAYlDFTZuwEseezsrgiqYiZ2gghnxNCenmerxQEwQyxFdD/ATgZ4nXhVogLLvPJfp9P+1n2vcGng7sCdwZOBvCM1qmNea/3nqGbomsv/075F+M5fuj5UHXs3dgMaMGDh8Z3g+8IZU/0XJx/HiikgF6omAkFHGu043il7Q0Adojx0myIbl4nSo/Jjl7tkoV7UQnoLG5iMCYGJqAzGAxGfsnWilBtYqqMWgV0GbXNW23zLQkopaCUGuVEjzLJIwjCFACGSCRyPQa+vwlFoqeVELJV+neQ47hejuOCg6345nnezvO8gVL6KYBnINqyzQOwAKKYnoZoVVgHoC3HUx00yWFba+sA8HjHDzvOaV7efBkAOC9z/td5uXPbWI8vxAVO7nnuvMy50brK6u24quNIuSf6OET0QiWDJkvlQTGNM96+gXIVxV4ALwGoQr+YfuqDDz5In3322URtbW3EZrN9PO6zLRwXArgY4mIAPUSrQR7Ateh3pBCkxy+GuJjg/sKfJoPByDNZxQSEEEopVaNrF6BeIVmtcbJaKKlF1pRSjSAItsEWFQNAKpVak0qlTlY+BUBQETft4zhO/rccM8XF4tKBxOPxhYIgeAA8C1F0nwsxbjoF/cLgbIiLkffrRZ6L6SpvcCZOmPLYlOdav9Z6kv92/ykAoG/Q76v/R/2TY3Ds6hNQQ8+HquWe5+5fut+Jvh6tCtwbOEruiT5GEb2QFu75phQr0IeycM+WIID3pM0AsW/6HABLotHoitWrVwsHH3xwoKGhAYQQPaU0OdzBJhEsbmIwJgAmoDMYDEZ+GfGiT9EPW43JILUK6CWVKMgSNc65KJASPfbBKiEAyNXjesVTBPQnemKUUptOp3shQyDPRZKGol8wfAHANIjVFXYA34a44norxOr0XIvpA7AeZ/VyDi7Id/KVAOA4x9E0jsNRoVfQJpuTVeXfK39Ltm2vvr16c9u321YlPks48WWMR0AvpQr0YkjQZEuuk06d0rYRgM1msy03GAzzXnjhBWcqlbqIEHI0gKel7YNJ3N/vfgCvQkz0aBQbkTYO/QkhG4ANE3CODAYj/2TT9kreT21xg4xa5z5Z/37lC7XNt2iQFhWbB1tUrBDJrRlPi8niOABwHLdDo9F8LovjHMeFFPmg8RCCaNf8LsTrpaUAjgCwCsBK9Femb0duxPRBf4s4Eyc4znF8HP80PhsAbF+ybRmDeC5DASD2caxKU6nplXueW0+weoWE8Hb3H7sP53t5HWcaU9zJeqCPnvEuBh4NuZxTAuJnfysALhAIzF60aNGKN9980+XxeAiATkLI8xBjphcopb3DHWyCYXETgzEBMAGdwWAw8guzcB8etVbfq3XhAKPAUEoJpdTM8/ygFoFS7/HMRE9UTvQQQnbLFRCEEFkcD8uJnng8flQ6nV5uMBg+yPWpD3J7D0TrNR+AzRArLA4GcBRECzPZ5n281dsDxpZ7nvN+3mk5zvJudFN0YcsZLWfLPdHHcHwIYUELHprIq5EG2wm2DqInNLIxUskHeDuN064xnn/fKY/z+SNRitUUhRTQ82EVGDrnnHOeP+ecc54/66yz1mzatKkJYjXCRQCuB3A2gH/kYdxxQymVLeoZDIa6YRbuIyBV36tx7mpdOKDGOU8olFIdz/P2YazV7RiYR+cByDFTJ8dxu5TV4xqNJkgISUnHRiQSuVGj0ewyGAy5bEs12DVsCGJcdASAvwCoAXAggFOlc94FMWbagfGJ6fuNHXwyWOe70Xeafqq+lZhIsut3XWuIiaTlnuijoO/zT4PUKHQL1vhH8TLzkeYuISJoIusjjQBycVXNBPTRjYMCjpWPmEmor6/fdvfdd2/76P/Ze/Mwyc3q3v/zSr339DLTs3psvIPxjg3YGNsYbGyDMZsJhCWEbBdyk0ucxFlvbhISyCXAQxaS/AK5SYCbBcgvCYkxYBuMAS9gQ8CDPbbHs689a+9bVUnv/eOVptQ11V3qbklV0ns+z6NnerpLOnpLKtV7zvc95/zgB/1vfOMb/1u5XP448Hrgn4HvAS9JwW4iiN8kCM1BBHRBEIR0iV3CHQud5Mhqa+vGHmDbuG0abybZI77vd4TZ44sEetzILpUgA2JMKXXYdd1tUYHcdd0xpdSy+3onQKPjhYI7sUAAACAASURBVH/fHWxhZnqtmB6uMj+00hPa/679rwl7ng/dOfT0xJcnng57oj/vnuf9s9sfO6viZNCh49yO6TU/v+aRE3994prhXx9+Wd9r+nYc/8Txl7ef2X507W+sfWIFpysZ6Muzk0X/u0yyNnzf94EtWusPK6XagGuBHyRtRyn1s8CngMlgIU7Sx28PfgxFNd3CWfSCIKycuD6Bxs6qXWCvkAwWjTuhbGShBq218n1/1UKZ44Hf1F2z22SktPpwpKx66DtN1yutniGL3SvhiU0C3wm2foyQfhHwZuaL6c9islrjcsrAJ+6e2BT2PD/jC2f8m+pU/v537r/t+MeO367a1RdXv2f1riUeXwOs/e21Pzh81+HeYx85du3QLw09Mv6v4xeU95bXrr1z7cPtp7UvdwFA0XqgF01AT71CmOd5ju/7Za31B4APKKXOwCw2SZS0fabAhvhNgpAiIqALgiCkS5ygvJRwtygoEmDjZFbm8EtEa+3ECPR01ew2EQn0HKwJ9Iy3QKAnaXxgV7B9GTgLExi6AlO6cIRqmfdGYnrdoEDXFV37eq7p2RX2PA97ok98aeJ8Z5Wz3JKE9N3edwh45PhfHr927pm5zR1ndhzZ+NGNjzi9jrfcYyICeqvbIW1bvu8rTEAUrXUFeDBpG0qp04CPYlonDCR87FWYLJBXYAK94XhKSqkh4MNa60eStCkIQksQBn0bYbOIbOvYxYEQFiUord7VYFFxH/PjLaWIz3RAKbW1xm+aUEqtZE5eSyt8dsepiukDnCqmP0c1M72RmH7K/Lnzos6x7pd0P7XpzzY9GJZtP/0fT7/n0C8cuqHr0q5lV9hy+93Kho9teGj4zuHrj3302CsA1t619lu9r+w9utxjIiXcl2uHDG2laqdSqSil1Ml4rdZ6H7AvSRtp+kzB8cVvEoQMEAFdEAQhXZbSA70VnKqssXXsto5bCIgEehbrodfH/HtkLhLo2aeUerJOoCeLTNpms9Bz1Qd2Blsopl8EXInJwg3F9KeA4bjG1v3muh/V/q7vtX3DgZC+FE4JOqieSLZ/O55qX/H1yyqbohCBkybYgZSz3T3POymgp8gnMcL8KHBHUgcNMub/AbgV835NYt6vNcHPXcDfJGVPEISWIuyB3rCEu6VlzMFuAd3Gcds45rpord1AHO9fZFFxR2QXn/mLivdGS6sHWxJ9wPPMGKYl1qPAIEZMvxAzr6tQFdOfI2Zmesc5HdOnf+b0r0V/53Q7/ua/2/zAMs5v3vxcuUrjVufQqmvFixtEQF+eHcimclcWArpDTn0mEL9JELJEBHRBEIR0iTvpszUwYLuQbNu4rRqv1rqnXC6fVZsJAYT/tkde7gPjQaBnxHGc3TWBnvEWDvRkmRkU15mOiun3AGdjxPQXY8T0E1TF9MORYxPz+Mth3vEnvz657vifHL+u45yO4VU3rto18ncjVw3/+vDLNn5k46OqY0XlO7O4Hlld8yIK6JlloKeBUupdmM/QRcCHEjqmo7X2gV8EXgicA3wC2Ke1vlMpdTnwf4HPaq3vTsKmIAgth5Rwb4yt/iLYNW6rMu6DCmWqXC6fprWuK5ADq2p2m4mI47siP4fbZAuXwk/yvBb7XCxl3jkKPBJsoZh+EfAWjJi+DVPNaxtQjhw/bfFZA/hTvnv4Nw+/zDviDa5535pHJu6eeMGxjxy7du2vr32o52U9y85sDyiC31QoXyZiK/VFx2kmH6ThMwXHFb9JEDJGBHRBEIR0iTsh01prG4NBtgroNo67VYMYS0ZrrbTWPZ7n1S0RGAR62jzPe4Xnea8IdpsOgztKqR2RvuNhoGeqhQM9i5HWPbzYe7HU98kHdgTblzBO5oXAS4DrgOMYIf3I0k9zeZR2lHqO/8nx69qf1340LNuuOpR34q9PXHP0j45evv731y+3Z3VRSrgXqsdexA5p20pTQFdKbQD+BPhNrfXBBNtBhAd6KfBQcOwBYJ9SSmmtf6iU+k3gN5RS39Fafzspw4IgtAxSwr0xto7d1nEXYsxa63bP8wZ83+9foLT6IODMzc39XLBLheqi4qOu626PCuSu644rpcoLWxRWSFRMX019Mf0pMrw/j37o6IvKe8trh+4cerj3lb1He17Rc/TwXYevPfaRY9du+stN9y6zD7pkoC+dTHyZiK3US7iTP58JxG8ShMwRAV0QBCFd4k76fAriJC8RG4VkciqUJkEurrPv+x0Neuj1A25kl4pSagwYU0oddl13W6VSuc5xnO93dHR8Nwj0LLtXtjCPld5DPrA92KKZ6VcBncFrrgQeI3lB/WQQpePcjumBdw58r//1/QfCnud9t/cdop2HOs/vnFihjSIEaQrVYy9iB/Jdwv0vga1a608mfNzwvVmLCcyCuSYdmGdtBXgcWA9sSti2IAitQewS7jFeU1RsFZJt9ZtaHq218n2/L2hH1b9AafXumt0mlFLjgSA+7Pv+eq31pq6urn8IFhdPJyw2tQppDCp8bi5mbyWfnxHg4WBbjfGZLgR+LDjuOCYDdjvVzPSkODk/H/yJwWcqhyu7e2/oPQbVnuiT901uXKZ4Hh4/bYomoGf5wcykhHuKGehp+UwgfpMgZI4I6IIgCOmylBLu1mWgR5zjQnrJMbB13E1Da+34vr9qob7jQaCnK7oLMBnJfDhYUyJwTCk1UxvomZycvNZxnONtbW0rLStnG1l+JjyqYvqXMOXdbwUuwYjox6iWeT+aoF0TDHr74N7aP/Td2nf41JcviaL0QC9UicCIHchpBrpS6i3AbcBlSR+b6ntyhOrzdwdwFrAZ2AOcCfQCsynYFwSh+cQV0K30mQK0xf3fbRp3SywY0Fqjte6KiuMLLCqOXptSxGc6oJTaWuM3TSg1v3f1zMzMKz3PW9vW1nYo0wEWgyw/FyPAQ8G2BvgJzJztrRjx/FlMmffnMAJeEmiAzhd2Tna+sHMy+ge3360MvGVg/wqOLRnorWsntJXLRccp+0wgfpMgZI4I6IIgCOkSu4Q7dgUGotg4dg1m1X6zTyRDUne2gkBPd5g9vkBp9T7m32+zkSyIfUqpJ+sEerIQ1/JM1oG+NOx5QBiE+TTmPrkQuBp4BUZAfwoTGFqumF4kcTttG1mWCCxU2cM0BHSlVC+mt94ngMNBmUCFyXQg+H9Zaz29TBPhe/IYcJVSaj3wBeBzwJ8ppe4F3gdMYBa8CIJQPGLNtYIqTjbNn6PYOnZbx53qmLXWbsRnOpk97vv+ABD6TR2RXXxMafXQb9pb045qzHGcVs4GbhVaYoHECjmByT7fC3wTk5l+EUZML1Et876d5Yvpad8TWQroaZOlgJ5VXCSLEu4O+fOZQPwmQcgcEdAFQRDSRUq4N8bGoEgRHOflsKLrXBPomVdenWqgpz2ySxjoGVNKjTiOs7sm0DO+gkCPYMj6s5uFPR8T+NmGKYN2LiYodA1wA1Ux/SlMlnpcsspCKIKALiXcl0kgoCdtYy2wAfhV4K46fx8Bvgi8eTkH11qH5/svwE7A0Vo/oJT6CPAbwOuD379Pa711OTYEQWh5wgz0Rtnl4jPZh21+04rHGywq7vU8L/SXTimvDqyq2W068JnGlVK7on3Hg23S4jZkeSSrufQJ4NvBNkS1zPvbMGL6sxifaQdLE9Ozmp9LBnrr2cnEVqVSUSkkSqTqM4H4TYLQDERAFwRBSJelZKBLOUL7sGncizpAQQ+93oVKBAaBnt6a3abCLAil1I4wyBMJ9kxJoKeQpHVN6wU5PKpiehuniulHqIrpx2PaSTtQk5W4nYWNIgWD8lzCfRhzv9fyW8D1mNYHce//BdFaH8K0Uwj//zGl1MeBTq31zEqPLwhCS+MjJdwbYauADvaOuy5a63bP80J/qb9OafUBzCLQkAowFvhMR13X3R4VyF3XHVdKJd3DWphP0e7hevPn48C3gm0tRki/CNMeqwQ8g6nmtZ14c9W8Z4eLgL4yW2mXcE88A52MfCYQv0kQskQEdEEQhHSJG+iwOSBi49hDx8OacWutHcAplUrnLtJDLxroKSulxjHBnsOu626LCuRBoCep/mpCRszunO12e1yvfWN7KfydP+s7s8/O9vZc1jMReelCznkzPzMVTBbFs5g59HmYwNDLgVcCh6mWea/nGEsJ96XZIAM7kEGJwIgd0rbl+37iwSCt9RwmGDoPpdRPAZ7W+ttJ2jvVvJ5RSj0P+CTwe1rrx1K0JwhCc4g7N7bRbwixdexWjTtcWO77/tq5ubmLtdb1sse7a3abCLPHHccZrskcH1NKTStlzVtYGKa+PzXQe2XvWO3vel7UM6ack9ezkc/UrMXkx5gvpodl3i8F5pifmV5v3lqkEu5F8M0gO58pE1uVSiVxkb7JPlNwCuI3CULSiIAuCIKQLlLCvTFWBUUCCpUVrbV2fN/vq+2hVxPo6QIolUrvwox/ItJD72CdQM+MBHpyQ6z7Wfua3e/bfTMe7rn/eO6X2je2l/xZ39nxrh03lfaU1l/wwAX/TE+6J3rib06cp0vaGfqFoW3h76YenFo7ed/kWes/sP6IalcQbzwVTBbFM1TF9IuA64BXYVafb8UEhk4E+2QVqClCkEYy0JeJ53lpZKAvRqrj0VqHx+8FrsVk1QmCUDD6+vr0xMREnOeJtT6Txf3fC+M3BaXVuyLtqOplj/cDVCqVVwW7lZRSo4GvdEAptbXGb5pQSmX5vS+sjFj389FPHz1z+GPDN228c+MD63523S6A4184vvngBw/esu5n13174/s3PpfmSZZ2l7pP/MWJy4buGvpB+8b2OQB/xneO/sHRK/ve1Le956U9S5k/H8P0Sv8msI5TxfRnMD7TTqpz2LTn5yKgt66d0FYeM9AXI/X3TvwmQUgHEdAFQRDSRUq4N8bWYBDkYNxBoKc7FMcXKK3ex/yxzEayIPYppZ70PG+z1vr07u7u/xMEelJ1iITMiH0PK0ex6Tc2PbzvV/bdvuOdO153zqfP+fKeX95z/cxTM2dt+o1N97p9rheIf41YtvOpp3T75NcmXwgw9AtD26YenFp7/E+Pv7z9jPbjuqLVEgT0KFExvZ36YnpUSJce6PFskIGd0FYWz6NMeqAHmWuZBIO01j8F/FQWtjDvn2Jp/TMFQcgXHlLCfTGs9JmCBbW5GLfW2g39pVAgDytvAaHf1BHZxQfCdlRjjuPsVUpNVSqVW9ra2u7t6Oj4geM4c80ZjdBMht4xtHf8vvFdw386/CrgAaffKR384MFbus7rOrDup9ftDF6W2udCz2q3fLA8eOS3jly7/n+vf8gdcMuHf/vwVeW95bWrplftCl+2jEMfBR4MtqiYfhkwS7XM+3KPv1REQG89O6GttAV0ldXio4x9JhC/SRASRQR0QRCEdIk7wbS5D7iNwaCWKeGutW6LBHrm9R+nGuhpj+ziU+2hN+I4zu6avuPj9QI9MzMzr/Q87zTXdcdq/1Zgmn59W42BGweO8XHu3vtLe9/wzE3PvAdg029t+sq696zbE/MQK3pPh+4cehpg8iuTF5e2ldaW95TXtZ/RfnzDhzc86nQ7m1dy7IAy8HSwtQPnY8q8X0/1c3QF8DgwkoC9WrIKbBRNQJcS7q1PGAgq4tgEQTDE7YFu6/zK1rG3xLiDRcW9nucNRPyl/ppFxatqdpsOxXGl1K5o3/FgmwwqC5zE9/3OSqVyi1JqQsTzQrGke9jpcPTZnz77G7ves4vhjw/fBNB1Qdfecz57ztfcVW6cudCK5tKdF3ROrv/d9Q8d+YMj1x7+9cPXq05V8U54q4b+x9CjvTf0HiOZz2RUTF9PVUy/PDjvbszC5J0kL6Zm+UwpkoCeVRJE6iXcm1C1K0vEbxKEBBEBXRAEIV3iTjCtLUdIiwRFMiaTlbtaa+X7fm8oiNcrr44p7xRlKsweV0rtiPYdDwI9U7WBHkFYCn0v7zvh9Doz3qjXDjB46+ChLO0P3Tn09Ny2ubWlHaWNABs+vOFRp9eJOpdJ3d9lTAbFVox4fg1wA/AS4GXAQapl3kcTslm4Eu4j/2fkXO1ptea9a7aHf5x6aGpo8iuTZ6//w/Xfi/SAXImtwpRw932/qMEShQmmSSaFIBSXOM8u8ZnsIyu/qd3zvIGF/CZMKVw3skuF6qLiI67rbo8K5K7rjiulyss5lUQGlB9sG29snA5HD9w28Nz0D6fPBei/qX97TPE8ETov6Jxc9xvrHjnyu0deBTD4k4OPB+J5SJLX7kiwfQPYALwTsyDlnZjM9KcxPtMukhFxC1fCvby/3HXs48detO5/rvte21BbGcCf9Z2jv3/0yv639W/rflH3ShMZss5AT1tAL+qiYxC/SRASRQR0QRCEdImdgY6UI7SJRDLQfd/viNFDLxroKSulxjDBnsOu626LCuRBoCetSbYER4pL7Gsb9jz3xr1Vg68ffHz8vvFLd7xzx+vCnugxDrFiZ3rqwam1lYOVNeH/Rz49cm7QEz3N51AZ2B38/LfAECbD4gbgJoyY/lSwrSS4kcWzNFMBvbSjtGb6kekXAKx575rtUw9NDR39wNGb3HXumD/pt7n97kqfWVkL6KlmbuQtA10pNYj5nvIw70246eBftNZzmEUoRV0cIAiCoaFPECziFJ/JLlY87mBRcV/Eb6qXPd5ds9tERBA/FLSmOpk9rpSaDsrLp4WN19oGYs85j3/h+OZDf3zo1Z3P79zvtDuVI3915Aanw/HDnugNWLF468/4zsinRy4K/z9x98QF3dd0Hwt6oqd5fx4GJjBtsB6jmpn+ImAGI6ZvZWVievEE9IPl7tLTpc3D7x/u3/jnGx9weh1v+K7ha0rPlE7vvqr7oAjo86lUKrnzK8RvEoTmIAK6IAhCusR1+G0NiICFY48EWxYct9baqQn01Mse74ruwvxAz8GaEoFjSqmZlAM9C5Kn/oXCkoh9TbWv2fkTO28Me56ve8+6PWO3ju0Je6Kf/8Xz/4Oe6svTONloz/MNH97w6MjfjDx/8iuTFwMM/cJQWCYzLWc9fK9KVMXyDuD5mKDQK4FXAweCv21leWJ6oTLQ139o/eNHfvsIY/80dnXpudLa2Sdmz3TXuWOn/cVpX09APIcMSgQGZBJE01pDduUVk+ALwJXAONV7KwwEacBXSvmYYFEHJnAqCEIxiZNdbp3fEMHmsS/mM6G17gp9pgVKq/fXHGMu4jPtV0ptrfGbJrLqjSsI9Rj54simsOf5OZ8952uqQ/m73rPrlcN/Ovwq1a3uW/vOtfuCl6Yyr/RnfCfseT70y0MPtW1smz3yB0euDXuit29sT812QDgnPBxsDwAbqYrpV1AV05/CLFRezvy3CAK6A+iel/aMrPu9dV8/+oGjNx76xUM3Oj1OqbynvH7Nz695qP9N/QeSspPAceKQRQ90J20bKSB+kyA0ARHQBUEQ0iWczCxKMMmxNSBi3dh1qHD4/ppSqdS5QGn1Pua/L7ORQM8+pdSTdQI9rewA2JaBbtt4G6IcxaprV+0ZuG1gW9jzPOyJPv718TPcPtcLepE1Ytnv7cSXJ8452fO81/HCnugzj8yc7b3b2+72uY0OsRLqBVFKwJPBFhXTbwRuBvZTLfM+HtNGFuJ22s+ak++VchTr/2j94/vfuX9o5vGZ8wBO+4vTvu6ucZdTFnUhW9IDvXn8EJgG5jC+qbPA5mOykVaaPSMIQusS57slzLBSFrYUss5ngpMLw9rL5fJZ0exx3/cHgNBv6ojs4gPjgd806jjOHsdxxmv8ppbtLW7hfW0LS/rs9l7ZO9J3Xd/WM/74jMfDsu1nf/rsb+y7a9/UqpeuOp7OKVaZ+f7M6sr+yppIz3PCnuhT35jaMPj2wbSfRfXm58PB9nVgE8ZnuhAjpk9TFdP30Pj7JMsM9LQ5+V71XN0zsva3137jyO8duQVg9U+ufrT/jv79SdvJAOmBXh/xmwShCYiALgiCkC7hSsBGk00pR1ggtNZtvu/31yuvTjXQQ7lcvj3YxafaQ++E4zi7a/qOjzmOE6e8tSC0NBt/aeO22t8N3DhwbODGgWP1Xl+HFT0r1n9g/ffwUdGe50N3Dj3tnfC2u33u5uBXzQpcRsX0TuqL6WFm+kJieuF6oANMPzI95B3zBsI/jv3L2JnRnugJ2Mpi8VFWJdxzFQzSWv96s89BEISWIW4GOmQbyG8ViugzobXu9Twv7Dt+Snl1TB9k5ubmzgp2mw4XFSuldiqlasXxyYKI0IW61sLS6DijY/asvzzrO9HfOR2OPvPPz/xuzEOsSCDuvbb3eNeFXfdFF6x2XtA5WbOItZmfs0PB9jWqYvpFmOzcaYy/tBWTmV7vPLNqe8UC9pO2YxI0Zn1n7PNjLwz/MHnf5PNXvW7V/rAnelJ2MkB6oNdB/CZBaA4ioAuCIKRLGCiPI6Db6iTnavFA0EOvNwzwLFAmsLdmt6lIoGc7UPI87/q2trYvt7e3P+04zlRBAj2NsPUeLzqJ3rtBplEqON1OXfEyCARlkUkB8d6vOeBHwdYJvAATFLoJuAXYR1VMn6hjI00yDThFe55v+sSmrx//6PHLx/5p7GowPdETslWkEu65Cgapmr4iOs0HgCAIrY7H0gR028idv6i1bvc8b6DGbzrpOwEDmFKzIRWqi4qPuK77nOd5L9Ba666urn9zXXdcKZVUBZpWRb4Hi01i1zftKVO9ak+R3zUjA30homL6aVTF9BcDU8zPTK/9DilCCXcFaH/Wd8Ke50O/MPStts1tU0c/cPTG4fcPv2rjn298IAERPatFx5nYymMGuvhNgtAcREAXBEFIlzADvRFWluQL0Frrlhm77/sdkczxgdry6pgeetFAT1kpNYYJ9gy7rrutJgtiXCk1r0+v53kDMzMz1zuOc9x13cksx9dEZHJfTBL97MbwAbMQO1vt+HPAlmCLiuk3A7cCe6lmWRQuA330M6OXnux5vsYtr/8j0xN94j8nLht428Bud3DFfdCLVsI9V8EgCfwIghAhrs8E5pmam2ddQrSUgK61dnzfXxX1m+pkj3fX7DYRaUl1KPh3PPSblFLTNfoA09PT64HOtra21MtWC0LOWOx5kMWzotV8JoCDwXY/sJlqmfdQTA9bY2UxL89UQJ/+5vS60rOlzWt+fs1DYdn2sCf6+L+Nn7nm51a88LhQJdwDn2mlfmSmiN8kCM1BBHRBEIR0iZZwb/S63GRhJ0xmwaAg0NNXE+ipFci7as4tGug5oJQaj4jjY0qpmdpAT5xTCf5tmSCYILQYzXAOs8pAXwlRMb2LU8X0ucDOKiCtxTmZCugbP7rx2/ioMOMl7Ile3ld+MgHxPLSVZQZ6qtkUectAFwRBiBC7hHvQAz39M2otsvSZ0Fp3hT5TPYEcs6g4ej5zEZ9pv1LqqRqBfFwptZzvwJZaOJAhNo65sDQxYSCtOW4e/KYDwXYfVTH9IuAlwGzwms3AdtJ5nzIV0FfdsupwxwUd/9lxZsdM+Ieeq3tGNv31pnvaz2ifWWT/JdlJ4DgtYSuPJdwFQWgOIqALgiCkS7SE+2K0VBZ2xiQSFAkCPd11Aj3RLIi+GluzkUDP3uDnqEA+scxAT1xsu+a2jbfwNGkRdNqBoLwcfxZ4Iti6gAsw5d1PB34VU6bwKUzZwiTF9EwF9HoiuXIU0eBQEnYSOFZL2PJ9X4JBgiDkFSnhvghBu6dExq21dn3f76+pujWgte4HQr+pI7KLD4wHvtKo4zh7goXEUb9pLolzWwCbrrdkGApxiVvtME37aZG0vxGK6WFm+nXA84F3YPykMDN9X4J2sxTQfaCuf5SQzwQZZIVHkBLugiC0DCKgC4IgpEvcDHSrS7gTY+xa67Y6gZ6T/fOCf9sju3hUAz0nHMfZFWQ+RMurl1IaUyNsDP5JMKi4JHltG30mivCZSeOzMAv8EFOe8DiwC5NhcSvwWmA3VTF9KgF7mQnoBbEDGZVwlwx0QRByTEOfILKw1cbKXXF9JrTWvZ7n1fYdj2aPr6rZbTpcVKyU2hn6SxGBfDIQ8JuBZKAXGEsrSSRFnBLuec1Ah3TOXQP7MQuQnw/8A3A+psz7S4EJqmL6/oTOoQjttbK0AxmI9ZKBLghCXERAFwRBSJe4q4KtLuGutVae562qyYCozR7vrdlvKhLo2R6WCIwI5FNNDPQ0olXPK22si5BYQJGuaRECTWEvtx8GWzcmM/0ijJD+WuZnpi9HTM80Az0jO6lmONTYSltAz1U2hVKqS2s92/iVgiBYQJxnsY2LUEM0oHzf7wgXFdcTyIEBwI3sV1FKjQFjSqkjrus+F6265brumFKqlfvA2uo3CUKrk0X/8LSPvx/YAdyLqeIVlnm/CiOmP4UR1Jcjpmdawj1lG1naycSW53lKa93K332nIH6TIDQHEdAFQRDSJXYJ9xivyS1BoKe2RGAY5Fnt+/7LZmZmronsUo4EeoZd1322Tg+9XE12F6Cw11wQUiJLxz1psgqiRI8/A/wg2Ho4VUzfTVVMn455/CwF9LSF7WaUcE91TL7vp17yMGH+UCn1feDftdYLlv9VSq0HPgL8ktZ6LLOzEwQhS6wv4a61dnzfX1Wn7/iA7/sbAKanp3+rZreJSEuqg5HFxONBFvl0AbJ8cz+AJZDXee5KsOH6Zj3GPPQob3T8LD4LOvLvvmC7FzgD4zNdDFwNjDM/Mz0OmZdwL4idTGwFba/y5DOB+E2C0BREQBcEQUiXwpdwDwI9fXUCPdHs8a7oLkQCPVrrWaXU4fb29u+GpdWVUjMFCPQsSCQzvriDPBUbg0FCvmiGwJ00iwWbpoH/CraomH5bsO3CBIWeYXExvYgZ6FLCvXn8LPCrwEeUUn+ktR5f4HUDwE8Av5bZmQmCkDVxfabwWZcrgtLqXaHPVCuQBwuL+5n/HsxFKm6NaK0H2tvb/91xnNFAIB+PlLUvKoVeaL4INo5ZSJ405515N9PEIAAAIABJREFU95kWs7s32L4KPI/5YvoYVTH9QAwbkoG+dLIo4Z6rql0B4jcJQhMQAV0QBCFdfHJcwj0I9HQvEOgJe+j1Md8BmY1kQeyN/Bxmj09EAz1TU1M/7zjOkY6OjmczH2DzsFVMlkBQMUmrP12W9rIgqxLucd6fWjH9hZjA0OuCLRTTn8ZksS/HxkooooCeVQn3vAnoGtNu4C7gCqXUT2mt6wUjwzlS0YUiQbCZOM+ull2EqrV2w9LqtVW3gNBv6ojs4gPjoTjuOM7uYCFxNIP8ZIbZ3NzcZeVy+ayOjo4nLRDNo+R13rdcbBuvVaTQYq5Z90sRMtyh8funMe2v9gBfoSqmXwq8DCOmh2Xea+evWX1PFVFAz6KEu5O3Eu6I3yQITUEEdEEQhHSJm4HelJX1Wus2z/P6owGe4N9+IAz4tEd28agGek44jrMrEugJBfLSUk+DFgyCZYRN45ZgUDEpUjnCrHqgt1q/wGng+8HWy6li+k6qmekzFEtAzyQrPCCTEu45FNB7gTuBFwF/ADyqlHqj1vq/al7nYN7DPI1NEISlEaciV1MEdK210lr3eJ5Xr+946DutqtltOpI9vjMUxiN+0+QSxbSWXTyQATaOWRBWQhZz6WZV1Urq+CzRRq2YfibGZ7ocuAYYpZqZfnCZNpZDVsJ26lnhEbIo4Z5Hv0L8JkFoAiKgC4IgpEvcQEfiJdy11sr3/d46fcej2eO9NbtNRQI92yOBnlAcn0pp1bRtQRGbA2CCsBjymVg5K3lGTwHfC7aomP56qmJ6L+Cu8BwbUcQMdCnhXh8N9Gqt/0wptQX4NPBdpdQ7gX/VWodjyWQBgiAITaWhTxDJvE60cpfv+x1h9ng9gRxTWj363VdRSo0BY0qpw67rbosK5K7rjiulks5ss9V/sNFXBDvHXGTSuJ7NvEeKcn8ud16ugd3BtpCYfniFNuJSxAz01MV63/dVxM/IC+I3CUITEAFdEAQhXWKVcA9E6SUFgoJAzyklAiNZEAM1xyxHAj3Drus+W1MiMI1ATxxsDIrYmo1t23UW0iGvGeJZZaAndfyomL6Kqpg+iPl+6aWamT6bkM2QrAX0LIILUsK9PgooAWitv6GUuhb4F+BzwP9WSv2B1nqO6nwmT2MTBGFppFLCXWvt+L6/aoG+46Hf1F2z20SkDdXBiM8UiuQzSmVfPCz417b5tG1+k23jFfJLXnyahY5PQjZ8TPurXcCXqYrplwR/fx/wJCY7/VAC9mopooCeSQl3IG8l3MVvEoQmIAK6IAhCuiyrhHsQ6OlbINATZo931ewfDfTsrxHHmxXoiYPNArpN47YxGGTL9c3y2hbhPW3pYFPlcKWjbUNbqeZ3pbYNbY8DjwM/jxHMO4A3ALcDOzBi+rMkI6YXMQNdSrjXZ16JRq31PqXUdcBngN8CXqSUehfV9y9PYxMEYWnELuGutVbBv2itu0KfqZ5ADvTVHHcu4jPti/wcbhMt2mP85Nhb1KdLC6t8xci1tWbMlpGlAJmmvSzuz1ZrezWPytFKhzvklpRTPZQ/66Nn9T530N0F7AXehBHWrwSuBUYwPtNTwPBKzyEgSwE9q+/G1MeU0wx08ZsEoQmIgC4IgpAu4eQmKo6jte6OBnp8318LtE9NTf1MkAVRG+iZjQR39tYJ9Ey2aKAnDjoMgtmGheO2bbw20IxrmtcM8SzeqxXZGPn7kXNGPjlyw4Y/3vCfva/oPQYwef/k+sO/c/j2oTuH7h98++BezPtzGJNh0Uc1M/2NmO+87VTF9LkVjqOIArpkoJ/KvPPVWpeBdyilngJ+F3gM+P3gz3md6wiC0BiPmu8xrbUbllb3fX/A87wzAObm5l4/NzfXE/hNHZFdfGA88JVGHMfZHek7Hi4sXu53U1OJtNGycT5t45htwcZF1kUgqwzxtI+/rDH4U767/13739h5Qef+jX+y8RHlKPxZ3zn4cwdv0rO6/fTPn35PRFj/MvAl4GyMz/RijJh+gqqYfriOmaWMpYgZ6KnO+T3Pc7TWectAB/GbBCFzREAXBEFIF10ul0+rVCqX+77fC4TZ4+2R13iYiU1FKXXccZyddQI9pbpHLwZWZRUESKBAEBZnoc9Inp8VLV/Cve+1ffvHPj82Mfzrw2/Y+JGN/6FL2jn8Pw+/vm1d24lVN68KSw5GbUxgnPTHMGL6hZjA0Jsw323RzPSlCBZFFNCz6oGuyJeA3sYC74nW+kNKqScwWRWfCX6dx0CXIAgx8H2/vVQqXe77/qWR7PHempeFVU48x3F2RPuOB37TVERoLhpWCuhKKa11US+pYBFpfW4b+Ux5/fC0dAl3p9fxVt2y6smxfxi7bviXh9nw4Q3fOfjfDt409/TcmUO/MnRfIJ5HbfgYv2gH88X0lwDXAccxPtNWli6mF1FAz6oHet78CvGbBKEJiIAuCIKQLjroubdOKTWilNoeDfS4rjumlJqemZl5i9a6q6en54vNPuEmYJ2AbmkGSV6d9+Vi23iTopmfiawCTS17/LYNbaXTP3P6f+z/yf1vOPT+Q28FaD+9fXjzZzff3TbUVm5gYwL4brD1c6qYHmamb6OxmJ7VtchE1A7IMgM9T9kG+4Hphf6otf6SUuoa4IvAC7TWeRqbIAhLQClV9jzvNOCoUuqw67rbavym8UqlsmFubu5nOzs7721ra1tJxl4esdF/AAt9xQAbxyzkhyzE1Jb2p9f+ytqtAGP/MHbdrpfvugQHf+hXhu4bfMfgnpqX1o4jKqbfQ1VMfylwPVUx/SngSIxTyUrYTl3UjpBFCXcnhyXcxW8ShCYgArogCEK6+J2dnc90dnY+jBEXFnwd9jrJtgZFwL5x2zZeW8g6uNHSwZRFyKqE+6Lvz/h/jJ82+unRF2/+7OZ73D7XA5jbNtc7/MvDt2744w33dV3cNbH6Z1Z/5+gHj94OsOZ9ax6tEc/jBDTGge8EW1RMfzNGTH+Oqpher8JK1hnoWQQXMglu5a2Eu9b6eTFe86xS6iXA5RmckiAITUIpVert7f0qJgNvodeEz2tnodcUGFsFdLBvzHmd6wqNkWsbj6xKuC+IP+M7B95z4NaBtw080f/m/gPh74fvGr7K6XPm1v/e+h+u+e9rnhn7h7HrAJx+Z3Lgxwei4nkcfyZcZLwdk5l+DsZnugojph+jKqYfXWQsRctAz6KEe+56oIvfJAjNwUanQxAEIUs08QRijb3PZBsFdBsdZxvHbANZf3bTtJdVD/Sm9gv0p/z2ua1z5+1/6/43eROeO7dtrnf/u/b/ePlgeZ2e1e7k/ZPrj3302C1ta9tG3CF39MgHj7x26ptTa2tsLGUMoZj+t8CfAl/HlHu/A/g14G3AxczvYVvUEu5ZCOguORLQ46K1ntRaP9Ts8xAEIVXiLCjWcLJdhW3YKqCLDyEISycLv6Zl21IlcXw9px1/3O858r+OvHn838Y3AwzfNXz1xN0TL/cn/c6w5zkOfsfzO/b6o37/8C8PX6P9k4dd6jUIFxl/Efgo8E/AAeBq4L8H2yuAdUsdS0IUroQ7BfSZQPwmQUgayUAXBEFIl1BAj/M624IhITaO3dYAmCAkQd4DqU0NBgVlBf/16AeO3rHn1j3v0jO6S2utNn9q8+e8Ua8j7Hm++bOb76aC2v+T+98Q9kTvfUXvsTg2FmEMeDTYBqlmpt+B6dEWZqaHwYwiCeipZ1IE5CoDXRAEIUKcZ6TNc2hbFw/Y6CuCnWMuMmlcz6LfI031mdxBt3L6P5/+xf1v3//GI//ryJtH/nbkYGl76cy+2/seXv+H6x8/+HPVnueD7xjcc+zjxy4Me6Jv/JONjyhHreT6hGL6c4ALnIvxma4BbsBko4eZ6VkK6FmVBE99TJ7nFVZAFwQhWURAFwRBSJdwgtlo8iwl3O3EtnHbNl5h+SzkMBchAz1tGp7/4DsG91QOVb564q9O3Aaw+e82/333S7pHpx+eXtNxTseBTX+56Wth2fbTP3P6fxx6/6GbnB6nEuyeVEBjFHgk2KJi+luoBjPOB7YA5XoHSICsBfRU7VQqlXA8EgwSBCGPeMTzmcDOyl02Lx6wbcx5XywqNJ8iZKCnTcPzD0X0nVftfH9pe+nMjvM6dm/82Mbv6opWzipnNtrzPOyJ7h33epSjILkxeJi2V9swOk6tmO4Bc8AQpn96WhSqhHuRM9AFQUgWEdAFQRDSRUq4N8ZGAd3GAJgEgopL0te2qJ+LrEq4Nzz+3La53rHPj70s/P+xDx+7oeuKrn/veXnPiZ6X93wl+tq2DW2lMz5/xpdrbCRNVExfjen5dznwOuAWqpnpz5GsmF6oEu7lclkEdEEQ8kxcn4kYrysito7dRl9RKC5JzgXjVjvMIy0j0B/94NEXhz+Xd5c3j//b+Ob+N/cfOO3/O+1bta8NRfSIjaTHUAGeDbZQTH8TsBn4ReAwxmfaSvJiepYZ6Kn7TZ7ngfhMgiDEwFaxRhAEISt8pIR7I2wce1EdXcE+kv7sxjlennuUp03DQE3Y81yXdfsZnzvjb9b/4fovlHaWnhf2RE/CxgoZAZ4Jfv4b4FvAGuDHMD3T3wK8EGhPwFboCxUiA71cLofjkWCQIAh5JI7fZKuIDPaOPc/zsmVjYal+Yek08x7JMhs5DWKdf9Dz/Jq+2/sePue75/x5+/PaD0V7oidhYwWEYvok8BjweUxp9+swYvp7g5/XJGQvdVG7BumBLghCSyAZ6IIgCOkSKwNdKeVb7CTbKKCH2DZu28YrJE+e76GsMtAXZfz/Hz9Xl3X75k9t/lz3S7pHu1/SPQr869EPHX3D1DemNvS/vv9gDBtZ9SYfBR4KtjVUy7y/FZOJ/iwmy2I7Joi0XDtZZFOknrUhGeiCIOScht8tSilrS7grpWwV0MG+MedZmBTqk/U9nPeFwVlkoC96/PJwuXPqW1MX993e9/DGj238LkDYE33s82OX9b+5/8BKbSREKAQ/E2xtmDZYF2EE9FcBw1R7po+swE5W44GU/SbpgS4IQlxEQBcEQUiXuIEOa0u4B8EgW4MiNo07r867YA9Z9UBP+7Ow6PHX/fa6LYM/Pfhs+8b2ufB3g+8Y3NP7qt5PRn+3CFkK6FE7J6iK6UNUxfS3ASVMX8CliumFKuEe9kDXWsvzVhCEPBK3apet2bk2+g9g72Jrm8Zs01iLQtMF7gRY9PjtG9vnzrz7zM+2bWgrhb8Le6KrThVH3M1ScI7aqQBPB1s7cB7GZ7oeuBE4RLXM+1LE9KxKuGdSISwQ0LMqSS8IQo4RAV0QBCFdpIR7Y6wbu1JWDTeKtQMvOFmKdXkvF5g2sd6fekJ5TPE8to0V0kjYPg58O9iGMEGhi4CLMWJ6mJm+g8XF9CwF9KxKuEsmhSAIeUVKuC+OrYsHZF2YUCSyupnznoGeNrHm5VHxPMQddJeyULfZPkaZ+WJ6mJl+A3ATcBAjpD+Fqfy1XDtJksm9KyXcBUGIiwjogiAI6RKrhDsmYGRbMCTEOgE9gq3jFopDGvdw+NzMmiJkoGcV2Gi2gB7lOKZP+reAtVQz0y/BiOnPUBXTa4MkWQvoqWY5VCoVFSlvLAiCkDfi+kxgZ+UumxcP2DZmET2FuDTrXsl7BnpWz5RmC+hRyhixfCtGTH8+p4rpYZn3sRXYWSmZ+GdSwl0QhLiIgC4IgpAuYZBHSrgvjK1jt23hgASCCkoKFRXiPC/TJM/3ahbPlFbIQF+IY1TF9HVUxfRLgTnmZ6Z7ZFQiMCCTDHQR0AVByDGSgb44to7dNp8pxMYxF5aCVo7Is0+WlT/Tqj5GmapY3kE1M/2VwKuBA1TLvIdieurtqAIy6YEuGeiCIMRFBHRBEIR0iZuBrgvqVMXB1qBInkW65WLjdRaWRqN7JM17KO37s0jidqsK6FGOAt8MtnVUy7yHYvozwJEE7MQlqx7oEggSBCGvNHxGKqVsFZHBbgHdNmwcs7B0FnsWZFVZKy2yOH4RfCZYuY9RYr6YHmam3wjcDOzHCOltK7QTl0wWOPu+L62vBEGIhQjogiAI6RK3FLHVJdxtXTxg07gjAU+heCR9bZt9r6QZaCpKefU8COhRjgIPBtt6qpnplwV/fyXwX8Au0gukpF7CXTLQBUHIOXF9JrC3ehXY6TPaOGahmDTbz0mSvGeIZ+XPpE2S71UJeDLYOpkvprtAP3A1RlAfT8hmLVn1QAcR0AVBiIEI6IIgCOkiJdwbY3MGuo3jtgkbrm8aY2yUTZHXTAoojridNwE9ypFgexC4HHgDsAl4JzBLtWf6TpIVvFN/3yQDXRCEnBPnmWuziGzr2G31mWwcs5Acec8Qh/z3QG/lEu5xmAN+FGydwHsDWzcBtwD7qJZ5n0jQrpRwFwShpRABXRAEIV1il3CP8ZqiYuvYi7T6PC42Xmchf+RZoC+ajbSfk6PBv5/FlCwMy7xfDsxQFdN3sfIgThYl3CUDXRCEPBPnGanBripOIRaXr7fRZ7JxzEWnWZ/bNO+lPAvcRVl0nJWdOcxC4wPA14AXYHymVwO3AnsxQnoSYrqUcBcEoaUQAV0QBCFdfGIIxEHA27ZgSIjNArpN45ZAkBCXRq0v8ipwFyVQk/cM9IXsHA62B4ANVMX0F2HE9KcxYvpuliemp17C3fM8yaQQBCHPNHxGRhYJ2Vi5y2YB3bYxgz1jFh9xZTTr/cu731QUn6kZduaALcHWRVVMv5mqmB5mpk8u0w6kPCbxmwRBiIsI6IIgCOkSN9AhJdztxLZx2zZeW0jSuVU0LuGeNnnN1MjKRpEE9IUyHKJi+kaqYvoVLF9Mz6qEuwSiBUHIK7Ez0LFzTmll9r1SCuy83kIxSdpvWs7fkiLvPcqL4DNlaadeNa1Z4Ilgi4rptwCvAfZgfKaniS+mZ1nCXSp3CYLQEBHQBUEQ0kVKuDfG1rGLyFFsbLm+RfrstlSvQG/Ua3MH3Uqj39WxUYRgUFb3VRyhfjjYvo7plX4RcCFGTJ+mKqbvYfEgTFYl3CWTQhCEvCIC+uLYOnZb5tRRbByzkC9axm/yZ3xHuUqrDjXvc+ONe21u/4J+k/RAT95OrZh+AcZvuhV4LWbhcSimTy1yHCnhLghCS2FrtqMgCEIm9PX1aeKtapQS7vZh27glECQkQZ4zbGOf+9S3p4b23LLn3eP/On56+LvSjlLP3jfs/fFjHzl2cRI2VkBRbIR2IH72wSFM378/Bz4F/AA4F3g38KvAbcBZ1H+2Z5WBLoEgQRDySlyfCeyMZdksoNs2ZrBzzEUmjeu52Lwy7xnosebN2tcc/NmDrz7wMwderUvV6hyHf+fwlfvu2HeHN+4tlDhYNH+m1ezMAj8E/hH4GPCfGB/lNRif6SeBFwO9C9hhCbaWRZCBLn6TIAgNkQx0QRCE9IkjjksJdzuxddyC0Iq0zOex6/Ku0bbT2o4d+eCR1wFf6rq868SBnz7wZl3Wbb039e5bZNesxlCkgBPLtHWIqqB+GtUy7y/GZFVEM9PD77lUywQGGehSilAQhLwiGeiLY/PYbRtzXheLChkSo51D2gJ301GOovfVvdtO/NmJWw78zIFXb/7bzfcf+YMjV0x8aeKlfa/re6xBBnoRWmtBtn7TcvyMGcyi4x8A3VQz01/LqZnp02Rbwl0EdEEQGiICuiAIQvpICffFsXXsto077NsY9jIUikMiDrvWWvm+34NxrBc7ZloBgrSDD7GP7/a53ubPbr7nwLsP3Hbk9468EcDpdyY3/cWmf+++onuswe5FELez6kmXVIbDwWC7H9hMtcx7KKZvBfoTsLMokoEuCELOafiMjMwhbZxM2iqg2+YzCcUmkbmg7/sdWutewE3ieMukJfym1e9ZvQu498Sfnbhlx0t3vA+g73V9j2344IbvN9i1CD5TaCcrv2ml44mK6T1UxfTbgm0XZvExCdhalGABivhNgiA0RAR0QRCE9JES7ouglNIxVk8XEcksEKxCa43v+6t83x/0fX9Qaz2otT75MzBINQh0PrAF+BEwGTlMnp8VSwo6uH2ut/4D6x/c/4797wbof3P/4zHE86KUI8xDBvpCHAi2+6iK6RdhBHQfU7rwKWBfwnalB7ogCHknrgDga61trNxls4BuI7ZdZyGC1rrd87zBWr8p3DDiI0AZeBLjN4VVjyCbhcFpE/v8V79n9a6xz40drwxX1gKs/931/9Vgl6L5M3m0Mw38V7D1AC/E+Ew3BH+/Ofjb0xjhPVGkB7ogCHERAV0QBCF9pIT74ticVWDjuPPcw1o4lZP38GICOTCotR5g/txzVik1qpQacRxnm1JqFFCVSmUj0AHcCLwasxL9CeCZDMbSMvdmaUep59D/OPR61aNmnS5nbvQfR6/rOLtjrP+O/v2L7FaUYFBW1yL83k3LViim3w+8HdiIybR4KTCByUzfSkJiuud5kkkhCEKeifsctNJ3UErZKqCDfWNumfmokBjz7uFAIB8Ifabg39V1BHIw8aSxwG867Lrus0qp0Uqlsllr3Q6cBbwIGMcI6VsyGlPL3KeHf+fwlZXhytq2TW1HvWFvbVjOXXWohc4xi2dKVs+tLP2mtOxMA98PtjOB92DG9bpg24lZgPwMCYnpUsJdEIS4iIAuCIKQPnEFdNsCAyG2jt22cbeMgy0sD601WuveMBsiCPacpbUenJqa+oUg2FNPIB9VSj2nlBp1HCe6zdXaqFQqmyqVigM8CnRhymBfCrwJk2GhMZnqDtmUqkuS2MGN0o5ST9jz/LS/Pu1fOs7vmDzw7gO3hT3RFxHRswoGFUVATyMDvR4amAVOAJ8BTqda5v0qqmL6U8D+5Z5P8NnJ2+dCEAQhJO7zy7Y5dIitArqt19vGMRcGrXVb1GfyPG8DQKlUek2pVOoCeiMvryuQR3ymicgCmpN4nteltT4E/CdwBsZnejFwLUZMV8Aq5lfzSoo0788lzc8P/87hK8Oe5xs+uOH7I58eOTvaE30BEb1o/kyR7IQxgnuAMaqZ6bdzqpg+u1wjQSUbEdAFQWiICOiCIAjp01BAV0pZW8Ide4MiIigLLUU9gbxOucDo3HEO43Rqx3G21wR6xhzHWbZDGzBLtazbACYodANwMSbTIixXeGiFdkKyCArEOr4uacfpd6bWf2D9A2HZ9s2f3XzPwZ85eKs/5zfqdViEZ0vWAnqWfQM1JuN8H3AvVTH9IoyYPk5VTD/AEt6HIJNCBHRBEPJK3Oedj52VuzSc7NtqE0WY1wiLk7t7OhDIoxnkq2t8pqhArjHzO5RSo67r7gv8ppHFBPI4pxH5OZxbfhXTCusGTPugX8EIjlswpbDLy7ATx34aNDy+9jV6TrdFe56HPdGnvjZ1vva0UhReQCcjO81Y4DwFfC/YVlEV09+AEdSXLaaL3yQIQlxEQBcEQUifOJMyDSitNUrlzn9cKbYK6GDnuFuqTLZNBAJ5j+d583qP1wR72iO7zIUZ5I7j7KiTQT47MzPzGt/3z+zu7r435dMfA74NXAnswAR/LgGuBo5S7ZfeqEd4M4n9ee98YefkmXef+e/R37l9rnfGF864J8buWfQ7LErAKavxhLZq5wO1YvoZmKDQxZh7exwTFNqKyUxflEqlIqUIBUHIM1LCfXFszUAH+8YsvlKTqSOQ1/pMq6Ivp5pBftR13drKWxO+7/fNzMzc2d7e/khHR8fOhE6z3ufCw4iJ/Zge0l+hWs3rdRgRfQtGeFzJfZZFBnrjFzqKjR/d+N3a369+z+pdgZC+GEXwZ5rty6RlB04d0yTweLDVE9N3YPymZ4khpksGuiAIcREBXRAEIX3ilnAHO8VFm4NgNo3btvs6c5YhkJeUUiNBgGdnPYE8htk07uHF7hWFcZ6/AdwHnIMJCr0C0zN9NyYotJVq+be4pP38zeL4FMBGaKeIAvpidjSwN9i+CjwPExS6BHgZZnFINDP9FDzPc5RSEggSBCGvSAn3xbFVQLf1ets45szQWrsRgXx1DIF8PBDIj7uuW1t5ayKoKLgYaV3PxXp8a6p9pQcxc8rLML7TJGbx8RZgeBl205yrZ+nTpEkRBfRWqRAWFdP7qIrpbwz2205VTK8bE/B9XwR0QRBiIQK6IAhC+sQR0MPJoQjo9mDbdQ6x8VonQiCQd/u+PxhukUDP6gUE8jCDfFdNoGdEKTWb84oXoXO8HejAOM6XAq8HbsM4zE9gVqO3inNchEAQiIC+HJwl2NHAnmD7ClUx/VKqYvpTwXYw3MnzPMlAFwQhz8T1Cawu4Y59c2lbfSZhBWitXd/3BxZaWIwR3U6+nPkCeW3lrfEYAnmrM4qp5vVt4DTMnPIyzLzyKMZn+hFBqfkWIc8Lm7O0QQZ2QltZ+UwswdYE8Fiw9QEXYvymN2H8omhm+kkxPWiHIn6TIAgNEQFdEAQhfeJMysLJoYN9fXhsFdDB3nELdWggkIfBno7ILuVIBnmtQD6qlJrJSCBPzJGO2deznr0SJvDzBMZxvgQTGHo7MIPpl/4EC2TuBmSVIZ728SUDPT5LDdCshOWWPawV08/EBIUuB64BRicnJ59+9NFHh8vl8gwSCBIEIb/4xHseW+k7RHokWzd27BuzLBpowDIE8olAID/hOM7OmqpbmQnky+x1njQHg+0+4FyMz3QDcBPxq3llkYGeJiKgL52lLAZeCSsZ0wTw3WDrp5qZHorp2x9++OH955133hOSgS4IQlxEQBcEQUifOEEemwMiOqZoVjRsG3crBAuaSiCQd9UI5KsbCORhBvkepdQTYfZ4xgJ51jQq4d6ICeCRYNuACQpdArwEOIEJCm0BRlZ2mksmK1G4CDaKmIGexJg0JrC5m4iY/o1vfOOK9773vZ2nn366Xrt27TGl1BXAD7TW1j93BUHIFdIDfXFs9Rdtvd42jvkkgUDev9DCYow4FiXMIB9xHGdXHYHcNqEszrzTB54Ltk7mV/N6LSZjdwv6uxGTAAAgAElEQVSm2leWSR5Zzc+L4jNlYSe01Sol3OMwznwx/ULf9y+66667bhweHr7x/PPP95588smblFJPa60nVmhLEIQCIwK6IAhC+sQu4a61VgUVxBbD1qCIrcJGoa+17/tdvu8PAo7neWfOzMwM1AR7OiMvryuQK6VGXdcdVUpN5+B5kMYJxl1wFIfDwP3A14CzMKUKr8FkWezDBIWewmSpZxEUyHt2eNEy0LNsm+IAlQSP5wO7gF0333zzVz7xiU+89Etf+tLV999//2pMr8sdSql/Af5Ra/1kgnYFQRDSInYJd621lHC3Bxt9psKPWWvt+L4/4Pv+aqBtZmbmlXUyyKP3+njEb9qdQ4G8GT7TUpgDfhhs/cyv5jWNqea1hWo1r6L4TWlStAz0PFcIGwe+4zjOdz71qU+dfvfdd1/7hS984TzgQ8D/Ukp9BfgC8O9a68UqLwiCYCEioAuCIKRPw5WTkVJetgaDbAsEhdg07kIEgnzf74xkQszLHq8VyLXW5/u+fyII9OxTSm3JoUDeaiz3DdMEYiNwD/ACjJj+WuA1wDYWL1OYBFLCPT5ZlgjM6tmUmq3Ozk7vjjvueHTv3r2jDz744Ma5ubkPAm8F/htmDvI/l3tspdRlmODSJcA6zGKTZ4G/1Fr/48rPXhAE4SRxs81s9R1sFdDBzjHnmkAgb5RBfvK6ep73oujC4qjP5DjOWA4E8lZjJfPOceDhYNuA8ZkuAV4KHMcI6WlSJHG7aAJ6FpUIUh3TJZdcsv+SSy753N///d//PPBuzL39VuCTwBdXcmzxmwShmIiALgiCkD4eUsJ9MWwOgtk47pamRiCv14O8K/LyilJqFAgF8h+Ffcjn5uZ+wnXdb3V3dz/SpKFkSdLObdoBgDImi+JJoBe4GJNhcVrw99dhAkP7Ej6XvPdYj9ooQsApSzuhrVSDTr7vK6VURWt9P3C/Uuq/M/+ZtRwGgb3AP2GyjnqBdwL/Vyl1ptb6j1Z4fEEQhBAp4b44tvqLtl7vlh5zrUBe24ecGoGcag/yUcdx9oYCued553med+mqVas+3qShZE2S806d8PHqcRjTK/1+4GyMz3Rt8Ldrgn+3YoTCpMhqUXBW4nYWNorkNyVVwn1Rgh7o+7TW9wIfU0qt0VrPrvCw4jcJQgERAV0QBCF9Ypdwj/G6ImJrUKQQGdnLoKnXOhDIB3zfXx1DIPciAvl+pdSToUAeZJBPLZRBPjc3h2SXL4s4b1qSn50pqr3RXg1cBZwHXAmMAj8CnsBkW+SFImSgF1FATz2r3vM8B7NoDwCtdRmzYGTZaK2/CXyz5tdfVkqdg8lwl0CQIAhJ4VP1CxZ7XvrYW7ULrbVVE8xIpTabaPqYA4G8r45AvnoBgXwyIpDvCwXywG8aU0rVbWMzOzu7OZMBFZMsnwUa2Bls9wC/jWlNdBummtdzGJ/pOSJz0RanCP5MEQX0NEq4n0LwXepH/n8igWOK3yQIBUQEdEEQhPSJM/ELAyLWBYOCoIhVgaAA28adSSDI9/2OBhnk3ZGXe0AY3DmglHoqDPQE29QKgnZND3xlRNb3cJr2pjFi458Bz8NkWLwUuA44iMlKfxIjui+HrDLQ857lHtrJKkCT+xLuIZ7nKbILWh4D1mdkSxAEO4j1jLTVd4jMSW0bu5XXm5THrLVWDTLIB0hAIBdO0gyfKa15Z3itv4MRzMNqXm8DZoGnMH7T3mUevyg+TdEE9CxbbJG2rSD2Kn6TIAgNEQFdEAQhfaSE++LYGhQBC8ettVYrycxehkA+ppQacRznoFJqa4ICudA80r5mGtgTbF8Bzsf0/rsZuAXYjgkKPcvSs3uLIqAXIWMjSzuhrVRLEdZmoCdJ8OB2gNWYPoG3AO9Pw5YgCNYSliNu9Gy21Xew2V+0ccwrIhDIazPIV9eUWI8u3p+KCOQHIgL5iAjkwiJMYoT072B6Pl+K6f8cVvPaEmzLqeZVhBLuRRLQC1XCPU0BXfwmQSgWIqALgiCkj5RwXxzbg2BCBK11u+d5iwnkPZGXhwL5qFLqkOu6T9cI5JMikGdClu9xms+KekGBCvB0sHUDF2HE9DuAEqbn3xZgd5196x0/TYrUZy/LAE2qwZkIWZRwTzMD/a+A9wY/V4Bf01r/dUq2BEGwk2gJ90avs65qF/YK6DbO5bXWiw97AYG8NoM8jkAeZpCvqOWLsDiR65nlws20j107lqPA14EHMNW8LsO0x7oe0w86rOY1HfP4aSIC+vIoUgn3NDPQxW8ShAIhArogCEL6xBHQw8mhrcEg2wJBNpafDNsUtFcqlYGYArlPUGJdKTXsuu4zIpALTWQG+F6wrcZkWFwGXA5MUM2wOLLA/kUq4V6EgFOWdjKxFWSgp5Uh9iHgbzDlB28HPq6U6tRafyQle4Ig2Edcgdi2OXSI1QK61pqVVLHKG1pr5XleVCBf3UAgn44I5IfC7HERyK2mWX5ytJrXl4HnY3ymW4BbMdW8nsBU86o3b83KpymCP5OlbwbZLDwuQgl38ZsEoUCIgC4IgpA+DSeZSinJQLeTwo07yCAfqM2E8H1/E8DMzMxdkZf7VDPID7uu+2yNQD4hAnlro7Vuxj2c1j2xlCDHCPDNYDsdI6ZfAbwcOIwR0n+EEdajZHE/i4DeenZCW6kGnXzfV1rrVAR0rfV+YH/w368GpQn/UCn1d1rrY2nYFATBOsIS7o2Iszi5iNgqoIdk+Z2dOkEG+ap6GeTAKt/3r5uZmbk+sks9gTzMIB8VgbzlSetz24zPxFL8gQqmYtdWzEL5izB+01toXM0ri7ZdaVJEAV1KuMc7tvhNglAgREAXBEFIn9gZ6E0So5qNrQJ6LsettW4LS6zXlgoMAj69kZdHBfIxrfWatra2u13XPSYCubAAjT4TrfiZCR3ke4HzMEGhVwGvBnZigkJPk10J97yL9FBcAT2LDPS0MilqeQxTmvAcQAJBgiAkQdwS7joIfNuGrQJ6dNy58RsCgbzX9/3Vi5RYdyO7zIQCuda6rJTa197e/ljgM405jlNq0lCE1mWxZ0Erfl6mgceDbQ3GZ7oUU81rHLP4eAswlcG5ZPUcLYqw3QwBPc8Z6LWI3yQIOUYEdEEQhPTxkBLui5FLITkBWs2hBU4K5LUZ5KsXEMg1VYH8qOu6z0WyIUaiAnmpVLqwVCqd3dHR8ZTjOHNNGZyQFknfy836bKw00ORhShE+C3QBF2KCQm8EbgPGMHNvh3RW1EsP9KWTel/yLG15npdaBnodXoW5j3dkZE8QhOIjJdwXIVK+3Laxt6rPRG0GeU0W+SALCOSO49RW3RqNCuSTk5N3Oo5zqKOj47nMByakTUvez0skCX/gBPBgsJ3B/GpeYTuszhUcvxFSwn1pZNKXPGNbWQro4jcJQo4RAV0QBCF94gglUsLdTjIf9wICeTQbYlX05SwskIcZ5EsVAm291kIypB2ISOrYs8B/BdsAcAlwDdAN/ArVDItDCdkD6YHeynZCW6mXcCfhQJBS6pOYrKDHMO0J1gJvBX4M+IjW+niS9gRBsJowAz3O62ydT4rflBGLCeTAoNZ6gPkx1dnAZxpxHGdbjc80tsQFxEUQWZeCrfd0mmTxniZ1n+4Ltq9iqnldgekd/UaMsL4FeAZT8j0pREBvTTuZ2UojA138JkEoJiKgC4IgpE/sEu4xXldEbA0EpTJurbUbEchXxxDIx4Ngz3HXdXcEwZ6RFQjkC55aQscRWosifXbTGssY8BCmesMFmADQJcDVmBJuW4JtLCF7IqAvzU6qonaNrVTH5Pu+o7VOOpPiEeCngHdjsukmgSeAd2mt/zlhW4Ig2E3YAz2O32Rj1S6w029KxU/WWqO17o22pqoRyAepL5CPKqVqFxWPplBhy7brXHSacT3TmnemNZawmtdBzILjx4BNwJuAMqYl1hZgF8nM37MSndMkq2ztovZAT9qG+E2CUEBEQBcEQUifpQjoNgaDbAwEwTKdD6216/v+gOd5C2WQ99XYOCmQO46zoybQM56gQC4IWZGHDPR6KEzw517gfkwPtEuB6zFl3fZggkJbMRnsyzk+iIDeinYgoxLuJJxJobX+DPCZJI8pCIKwAFLCvTE29n9frs9UVyCv8ZuiMdE5pdRI4Ddtr5NBvpy5mSCkTTMXqact0G/HZKUPYhYfX4bxnSaBJzHC5PAKbaRJETPQs4gdpb4owPd9MGMSv0kQhIaIgC4IgpA+DSdlERHTxmCQzUGwU8YdCOT9nuetjiGQQ1UgH3EcZ2cOBHJbr3WRyTJwk+b9k+W96WOCQtuBDuCFmIDQ64DXYrIvtgR/j+vYZymgp00RBfQsSrg7GfZAFwRBSBop4d4Y8ZsCAoG8Z4FFxasXEMjDHuQ76mSQt5JALpW7ikuW8868ZaAvxCjw7WA7DeMzhdW8jmJ8ph+xtGpeUsK9Ne1EbaVGqVQKRfqseqALgpBjREAXBEFIn9gZ6Fpr6wIiSimrAkGhQK61bvd9f93MzMwrawTy/ppdogL5rjoCeV4m/RIIEuLQ7GdB2uJzveOXMBkUT2AWyFyCCQz9ODCDybDYAuyPcfy0kQz0FraVRga6IAhChkgJ98ZY4zeFAnlQUp3Z2dmrgb4av6k9skspzCDPgUAeByuus7Aimn2PpC3Q1zv+wWC7DzgX4zO9ArgR2E21mlejlgoioLemndBWqouOy+VyOB7xmwRBaIgI6IIgCOkjJdwXp1CBIK214/v+gO/7g+FWJ4NcBa8d8jxvTUQg351jgTwuhbnWApDO9VzMMc9zNkWcc5/A9E57BFiPKVV4CfAS4ATVfukjixxDSrjHJ/Wy6lnaSqkHuiAIQlaEGehSwn1hCjP2QCDvrvGZVi8kkHued20kg3xXjUA+opSaVaoQb41QTLK+OfNctSuOv+EDzwVbJ9VqXq/n1GpeC4mxWfgzaVcDzMo3y6rXemgrVTvlclky0AVBiI0I6IIgCOkTZlMshpRwzwmBQN6/iEDez/zxTESCPXscxxlVSo2WSqWblVIHe3t7/7NJQxGEVqWZz4NWqpRwBNMr/WvAWZig0DXADZhs9CeApzBZ6iA90FvZTia2PM9TUsJdEIQcE+sZGbQnyo3vkDC58ZsWEMhr/aaOyC7lSAb5SYHc87w1nufd1N3d/XHXdRtllRaFVpqPCi2I1rFukbTvo1a5T+eAHwZbP3AxZhHy24FpjL/0BHAgso/0QG9NO6GtLKp2obVutXaHgiC0ICKgC4IgpE+cyV/4mlwERBImLF9PK2QN1ArktT31WFwg3xsK5I7jjLquO7ZQBnm5XK60wngzpFUc7Cyx5QJnfW2LnIFeDw3sCrYvAy/AiOmvBV6DybzYgukBGL4+LYoooGcVOJEe6IIgCIsjJdwb0zICeiCQd0UE8tUxBPKTi4qVUk+E2eOB/zRTzzcqlUoXep6HssxxokWus5A4RfCJWyEDfSHGqVbz2kC1X3ptNS8p4b48O1n4TVmVcJfsc0EQYiECuiAIQvo0nPwFfcDBzmBQdPFA6g5lIJD3LSCQr+ZUgXwyEuzZV0cgX65YUQTneTlIMEhoRKMS7s2y3QrHL2N6oj8J9GIyLC4FfgzTSx1MsGhvArYWowgBpyztQDYl3JWUcBcEIccspYS7jT4TZCyg+77f1SCDvDPy8roCuVJq1HXdUaXU9DI1cBt9JhvHLCyPhe6VLOa4rX78w1SreZ2N8ZlejqnmNQuMAd1Uq3mlQdEE9MKUcA+q2QiCIDREBHRBEIT0iTP5s72EOyTk5GmtVYMM8gEWFsj3JyiQNzxV7LreEggqJknfw838TGSVTZEUU8B3g20tcBXwYkxW+jVUMyyOJWgzyyBNVhkOhSnh7vt+lhn1giAISRO3IpevtXbTPpkWJVH/wff9zjB7PIZAXokuKlZK/SjoPb5SgbwRNldqswFbfETxmZp3fA3sDLZ7gAuAWzCLju8CtmF8pm0km5WcZQZ6UXqth7ZStVOpVLL0AQVByDkioAuCIKRP7BLuWmsbAwNLCooEAnm9DPLVkRLr0ayUqUiw50CNQD6aokAeBxuvty2IQ5Yeec2sTtNRP4YR0l+MCQxtxJQqvA44hOn79yRGdF8JRSzhnqWAnmpwy/M8kHKEgiDkl7CEe5zXSQZ6DCIC+UIZ5F2Rl1eUUqPASYG8xmeaanIVdWv8puB9tma8Qm5Ju61WGscvAz/ClHUHI6pfCrwVk5X+FEZM35eA/SKWcC9Ei60gA118JkEQYiECuiAIQvrEmfxJCfdgUr6IQB7NII8rkI8ppcpZDygmVgqsli4SKTpZ3stp3j9Z3JtZ9CcfBr4HfBU4HxMUuhmTabEDI6Y/iwkgLddGUQT01EsEZmnL8zzp5ycIQp5ZSgl3W+eT88a+RIHciwjk+5VST7aYQL4QNvpMNo7ZFopQ+SirB0XaflMF+E6wrcX4TJcCVwKjVKt5HV+BjbTJyjcL42+F8M+CDHSp2iUIQixEQBcEQUifONkU1pRwDwTyVWGgx/O8FwBMT0+/U2s9wKkC+XREID8UBnocxxlpcYG8EbYF/yQQJCRFmsGgLATurI5fAZ4Otm7gIkxQ6A5Mv/SnMUGhXcQfd1bPrCwz0LMKnqQ+JhHQBUHIObFLuMd4TSHwfb8jKpADHZ7nXTw1NfWCQCDvjrzcA04uKlZKPRXxm0Ydx5lSSuVxPm5rCXfbxivkjzwL9LXz8mPAA8A3gOdhfKargOuBgxifaanVvCQDffm2suiBLj6TIAixEAFdEAQhfRoG5yPBjNw7yrUC+QIZ5NG+hSUApdSs4zgHo4GeIBsirwJ5HHJ/vZeBjWMuMmlcz8Uc5jzfP1kJ9PVszGCy0r8HrKaaYXEZMIEpZbgFOLwCG0kiJdyXQdADXYJBgiDklTADvRGFKeFeK5DXySCvFciV1tp1XXePUmprQQTyWEgVKyHnZH3/5jkDPQt/Y6H3RwN7gu0rwPOZX81rO8ZnilPNq4gCehYLjzPpga6Ukgx0QRBiIQK6IAhC+sTugU4OgkGBQN4bBHpWxxDIZyIZ5M8qpUYigZ6xSqVyXqlU+rGurq4vOo4z16RhNQPbMtAFodXJIsiRRWC7kY0R4JvBthkjor8IuAYjoG/BCOoTdfbNMkhTiABNlrYkA10QhJwTVu0qTAn3QCAfWEAgX818gdwnyCBXSh1yXffpGoF8cmpq6pdc132yu7v7gSYNqRkUZqH5EijsQgghcZp5r+Tdb2p0/AqwNdh6OLWa11aM37R7gWMVUUAvRIUw8ZkEQVgKIqALgiCkT5xJZsuUcNdaU5tBHhXJgUEWF8ijgZ5Rx3FKjUwG/zZ97BljW2DEtvHaRJLXNs5zIK/3Uitma/w/9t48Po7ruvf83VvdQGNfSJHiqoWkRHEVF1GkFsuSrY2mnuVYkmXF4yzjZzvLTCbPTjKJP7ayvpn5vJdFmeQ9v4z9cWJlHHu0WLFs7bKohaJIcQMXkOC+gyBINHY0urvumT/uLaLQBBrVQFcBXXW+n099ZJOFun3R1c0653fO75wzx+sAFkCL6fcBuB/Aceik0EEYp5BxrjEewtiB7vs8P+5AZximxCk5C3ciitu2na+DvNJ1ulsgv2BZ1qFcgdxDB3nJFA8UkVJ97psoUXufo0Kx7ufJvD+CHks1FdboB/CxORox5OZ1K3TRsTMv/WLOGmERtoMU0IOYgS7BM9AZhvEIC+gMwzD+4+XBLDAROZ9ADqDezCF3//uQcgnkh3MSPV1F6BqPqoAO8J4ZZiQmy8I9CIv1ybJwHwsbwGFzlANYAp0UehTAJgCHADRhyKowLAK67wkaF9xNwTAMk58pZ+FuBPK6nLFUDXkE8i4TN7VZlpVbWNxTBIv1KAroDlHad1SLBpjiEYSFe6le31ljvNfvALDZHHOhY6bVAO6EdvNqgp6XHiYB3fk3NxQFzsbCnWMmhmE8wQI6wzCM/3i2cCeiCSeDiAhEVOV0Q4wgkNdjFIFcCHFkhA5yv23VoyqgRy0BxomgcOLHPezFupW5mmK9F4MAdpujDsBy6M70ZQBS5pzpAC4Vab2RCGMHuu9rcQc6wzAljicLdyEEFWseNhHF3DFTbhc5gCrX6UEI5GO+ZEQrfgCiGytGiSi8t1HYYylRrOfys+Z4HcBCaDH9U9BuXmno4uMyDLl5FZswdqD7XnRsOtA5ZmIYxhMsoDMMw/jPmA9/QgjPFu6jCeQ5yR739/ugSfQkhRBHR+ggT422VkBENSnCIiDDFA53oI9NMdfoAvCBOWZBd1YsBfAFaAHdsSvsKuKaQLACelCz1oFgBHS2I2QYplRxOtC9WLh7Kjo2ArnTQd4whkBOGBLI2y3Lyi0s7g5AIM+LWZ9jpmgQtfeZKS7cgT42xby+DaDFHAloN68HAVwP4JvQbl57oUdkFfNZPYwCehAW7mKy/z1nGKZ0YAGdYRjGfzx3oAMQRiCvtG17pE6IhhEE8rQRxzullMdG6CCfbIF8LJzu+ygmCXjPDHM1YQ1mp7KFuxdaAeyAFtBfgp6Zfjf0zPRT0EmhZgx1qU+EsHWgBymgczcFwzClitfvyCsico5APlIHeXXOz+UTyHtcRc1TlSgK6A5R2ndYn4WZ4hPGeyUoC3e/SAHYBWAtgHboouMV0K5efQD2QcdNrUVYK2gBPajC4yBmoHPMxDCMJ1hAZxiG8Z9hD38jCeTGXh2ZTObTmUxmE4C460fyCuRCiJQQJZ1PiGQHegQ7SMIY3DOaoN9bP7sdSvk+DTLZdAp6vl8ZgMXQFu+bAGyE7rzYC+Aoxp+YCGo2edACOs9AZxiGGZ2rLNyJyDICeYMTN9m2vQBAXW9v7zdwtUDebQTyy5ZlOXFTsoQE8rGIWvwAV5dgpPaN6O03EoSk6zXIbmS/CGo++SCA980xCzpmWg5gPbSw3gQtqI/XzSuMHei+O2pxzMQwTCGwgM4wDOM/KpVKLVVKrXB1kOcK5J0AIIS4ZFnWkRyBfKDEBfKxiGpSpJQDToZxKPbndqzrlfL3RFAd6H6SmzxJY8jGvQZ6TvoKAE8CGABwADoxdHYc64RJQHeshrkDnWEYZnRUJpOZmclklhBRhYmZalx/TwC6ob+7bcuydua4bnWHQCAfi8gJ6OCYiQkHkxEzlepnJ6ii4KBEZ4dWc7wB4EbomOke6JnpTnFyM7ToXugaYRLQg7Bwl0IIjpkYhvEEC+gMwzD+Q0RUBSAlpTwxQgf5gBACvb29T8disf3l5eW7JvsFB0xUBXSA98wwIzFWwFzKHeilNgM9l3yf3x4AW80xAzoptALavjCJIaG9w+M6QSVowjQDne0IGYYpZQhAjIhqhBDtUsoTru7xKwL5wMDA/bZtL66oqNg82S94EoiigO4QpX2XqvDJRINSnFE+GdcfLZ5R0E5dR6HdvG6Bjpn+A4DPQLt5NQE4hrGf64N6LwIpBjYEYeHORccMw3iGBXSGYRj/URUVFduhO/HyQRh6MI0MEbbli1QCLCR2dUwwhPVz4fe+grRwH2uNiwDeAvA2gOuhk0IboLsszkIL6QcA9OdZJxQJGtc6gM9iPXegMwxT4qh4PH4mHo+fgf53ZDQi9QydQxT3TgBARFHbd9T2yxSXUr5/gnLVmiwB3U0aWixvgnZcWQ4dN30R2s1rP3TcNJqbVxg70H23cDcxU9gdaxiGKRIsoDMMw/iP14dMFcHEABDdDvQoJsCAaO457ISlOCIIi/WpIG4HuQYBOGGOVwDcDJ0UehjAQwCOQCeFDgPI5qwTigSNgS3cGYZhxsaZgT5WQbHycE5YiWL8EJbnzEKI4p6jQpDvrZ+uXaV8ffcaflJoPNMD4ENzzISOmZYDuA3awctx80rmrIEC1xkPobJwt22bXbsYhvEMC+gMwzD+U4jQELWECBDdrgIgmu93lIjC++vHHifLwj0IppK4HfQaGeguiv0AqgAsBbASwOPQs/6aobsvTiO8Hehs4c4wDDM6XotqoxozAXo0WFT3HtV9MyHAp8+tU3QUVkq5sHmia7QBeBPa0et66JjpDgCfBHAGQ25eQQvoQY2+8t3CnWegMwzjFRbQGYZh/MfrQ2YkLdzBHehRIczBPVM8BEb/XARhgV7KnQ5BdVIAE/899QHYbo5pGJqXvgpAF3SMcnmCa3ghbBbuvq/BMAzjIwoevpPNWKAoPUO7ieLeoxorRmW/HCOOj7HuDz+fcYPqQPeTqS6gO7jdvH4B7ea1EsBGaEevC+Y8v/OIobJwNx3oHDMxDOMJFtAZhmH8x2uiQ3k8L2xENSlCRJHMF0TlfY7km8vkJSixdqp2oI/GZQDvANgMYB60kL4awDJocX0vgH3QonuxCVUHOhFxBzrDMCVLTU0N9fT0eImb2MI9WkTxmTqKe2YYh6DE2lIQ0N3kunktA3C7+bvfg3bz2gvt5lXsvQUtoPvegQ6OmRiG8QgL6AzDMP7DFu75iayAjmjt2bHqn+zXwRSfYr+po10viG6HUk7UBJnY8GMNgk74nIYW0rugExv3A3gAwDHopNAh6ARSMfB9xp5rHfi9Fs9AZxgmBHh5Po7aM7SbKO49krFihK36w05QwbCf908YBO6gPl9+7aEPwDYAAwA+B2AH9Lz01dAxlDMv/VKR1gsklnGtxTPQGYaZMrCAzjAM4z9s4Z6fSCZFDFHcMxMu+B72TlAW7mEQ6QWADgCvAaiAnpe+AsCvAEgDOAidFDoxwdcSNgt3TgYxDFPqePkOi6KI7BDlvUd130w4KPr966EwvVQr19nCvbA1AO3m9UsA86FjpnUA7gZwHjpm2o+JuXmFqgPdtm0uOmYYxjMsoDMMw/iP14c/tnCPFqUa0E6UqL3PTPEp1Q50v+e5ldIMdC/rOGsMQHdV7ADQAN1dsdIcPdD27nsBtI1znSDm3wXye+MOdIZhQgBbuOcnigJ6FGOmKO6ZKR2CmkhoKj0AACAASURBVIHud1wWJgGdzHHKHK8CWAQdLz0A4EFoN68mAC0o3M0rjDPQOWZiGMYTLKAzDMP4TyEd6FFLiEAIEWUBPUp75kQQM1Gi9HmZCGHpQB9pjSSA98wxB7rD4lYAdwC4iKF56d0TXKfYBGJ7yDPQGYYJATbYwj0fkdt7hGPFqO2XGR/5Rl9x/J2foMRtPxktxshCO3YdxJCb10oAn8eQm1cTgJMj/OxIBOKmZQjCwp2LjhmG8QwL6AzDMP7jNdHBFu7Rg/fMhIGiBbiTPO8xiA70Uhe3g3p/vPyuzpnjdQALocX0ewF8GtrafS+AZugk0UTWKQZBdaCzgM4wTKnjxZErciKyiyjvPar7ZsJFsZ4FJztmAkq/A91vguxAz0eum9cKXO3m1QRdjDzWOqGwcGfXLoZhCoEFdIZhGP9x7JTGQk2yeDRZEDDpwtlkEOUEGBMe/LiH83VS5Pt7P9cuheuHbQZ6IeNPDpujHMAS6MTQZwF8BsAhaDH9GK7umgjVDHTz7ygngxiGKWU8xUyIZtExEM34IYqxIncOh4+g798g1iv1uGkqxTNBrZEE8K455kLHTKug3bzaMOTm1TPCOihwrfHiezxjLNyzfq7BMEx4YAGdYRjGf5yE+VgPt9yBHi2ilhiJ2n6Z8TNZ3wV+r1vq13evMdWSQQ6DAHabow56XvoKAE8B6MfQvPTz5nzfLQJd68DvtVhAZxgmBLCFe36iuPeoxhBRep+jtNcg8btD3C/C1B0+ldc4aw63m9d9AO4HcBw6ZjoI7eYVSCxj8F3cZgt3hmEKgQV0hmEY/3E60L0I6FEMHqMqoAO8Z4YZD6XagR5Ut3PUOtBHowvAB+a4FtqmcDmA2wFchrYqtIqwjheCtHAPYjYhwzCMX4wZD5mZ2JF8nozy3hGtfUe1aIApLUrZwj2I6091Ad3BBtBijgSG3LweBbAJWkTvNeeGwsLddKBzzMQwjCdYQGcYhvEfBe92hFFKDDhEVUCPWgKME0Hhpdjv7VgW7n5R6laHUbVw98IFc7wJ4AbopNDdAOLQtu8XARwAkCrimm6CsnDnGegMw5Q6Xr7DFABBRBAiSo/SAKIXPwDRjRWZcBJk3FSqHegOpV7YXIprpADsMofj5rUSwHTz9/dDFyG3FnHNXAR8jpm4A51hmEJgAZ1hGMZ/PAX9pqOALdyjAwvKTBgo9ud2sr8HghCf/b4+C+ijo6BnoR8D8AsAvwudPPkMgIeh56jvBXAExU2qBGJ7qJTiZBDDMKWOl4Jid+wQtefpKAvoUSNq73PYKer7acb2TDalLNCzTfzYuN28PgngHgDLoN28LkHHTHvNecXE9xFbtm1LIkr7uQbDMOGBBXSGYRj/cVu4j3XeVAiEAsUUDgAR3Dt4z0wImITur1JN1vidRAlK3A5iDd87D6Dn+aWh7Qq3Ymhe+hcADEB3pO8FcKYIawXy3nAHOsMwIcBrzAREV0CPYsE1EK0YImr3NVNahKUomGMm73Sb//4NgBuhY6ZPQM9MPwUdMzWjOG5evr83XHTMMEwhsIDOMAzjP2zhnp+oCuhRK5jgRBDjlcmycM+3dilcPyzdGs46Qc7Y64UW0bcCmAGdFFoOYC2AJIY6LDomsA7AFu4MwzBj4dXCHdBCctRmmEYtfgCiGytGbb9McQniWbqUr1/q3eFBruFeRwE4ao4yALdAx02bAGyELkzea/5+vDFJEBbuHDMxDOMZFtAZhmH8p5AO9Ch2FEQ1KcKCMhMWJnQvE5Fl23adUqohm83eBB2MHwQw6Md6eQjDjHWg9JNNQa4zkkXgRQBvAXgbwHXQc/82QNsWnoOe+3cAQH+B62CEtYoKC+gMw4SAQi3cowZNEevmIOGYiQkT476fiQhEVGnbdqNt29OJ6HoAnQAuF+3VeYNjJm+ESUAfKWZKQ8dFTQBqMOTm9SS0m9d+aDH9bBHWKiq2bQsiyvq5BsMw4YEFdIZhGP/xmuSJYkcBYH4/EUwGAdF8v6O45zDj6f1USiVs225USjUQUYP5byMRNQCodV3H+R5cBi1S7oa2hfO81gQpZfE5THaHU2EdAnDSHL8AcDN0UughcxyFThgdBjBWAiYQC3elFAvoDMOUOp4FdCISkzBGZrKJarwIRGvfXDQQPjzdv0QklVL1tm1fFTOZuKnMdbqCfj69AB0z7cWQhXYpd6AHFdP4TVjmrHtZpwfAh+aYAV2AvBzAbSjczcv3PZncI8dMDMN4ggV0hmEY//Fs4R5RETmSXSRCiKh1kEQxERSZ99cke2qVUg3OkZPsSbhOTwkhkkKIDinlWSll0hwd2Wz2mkwmMx36GXU1dPDdAZ0UavZ5G6X+foUl2eSsE1QyyItFYBa6oOMAgEroAo8VAB6Hdkpohk4KncLIr5st3BmGYbwx5vekEMJt4R41IiegCyEiGSsievuNDEqphCtmulJgbGKmOgx/buwUQiSllGeEEHudmEkI0TcwMLAc+jl0EYAHATwA/Uy62+cthOHeDEt3+FQR0N1cBPAmtKPX9dAxk+PmdRY6ZtoP3aU+0bXGhW3bkjvQGYbxCgvoDMMw/sMW7vmJalIkioIyEL33OTQopcrcyR7TFTEXQFlfX9+3MNymusuI5K2WZTVLKTtcQnlqtDVs274GOhH0EYAt0Bbaq6AD7vvMaddCW7z7IUaWcvd2lLocJnOdfgDbzTENOim0Avo+7cZQh0V7zjoYx1qFwt0UDMOUOmzhnp/ICeiIZswUxT2HBiISprC40RU7zQGAVCr1FQDlrtMHhRAdRiQ/nxMzdbsKhoahlCo3BfknAewCUA1deLwKuvM3Dd2NXgPdHezLVn26blhctcKyhrNOobE3AThhjlcw5Ob1MLSb1xHomCnXzcv3GehKKY6ZGIbxDAvoDMMw/uM8/LGF+8hENQkW1fc7KpRc4sske6rdyR6nG4KIGqE7bx3SQogk9D574/H4h0KIpGVZSSlllxCiGAGp20L7VeiE0AMA7gJwK4A90B0WXqzgvBDEPL8g7otSTzaV0jqXAbxjjnnQicu10PdoK4Y6LIKcge5rwolhGMZnbLCAno8oxg9Rfr+jQMnFTMBVhcWO3boznqoew5/9uoUQ/QBgWdYuKeU5KWXSsqyOfIXFBdILXXy8BcB8AJ8F0ADg96HHDu2CFiqL8ZwY1Az0UrWgdwibgD6RdTLQMdF+AFW42s3rAHTcdBrBzECXSikW0BmG8QQL6AzDMP7jtQPdS8dF6IiwLR8QoT273mdmEiGimG3bDTkCuWO1Xo/hz4Y9pov8kmVZR0xnRNKyrA4hRL8QAv39/V8EgPLy8h3FfJkj/FkKWix/AMAb0AkhR6g8BZ0UOggdnI+XoBIQfhGWbo1SXeeMOV6FttJcCeB+6Hv2gjnHV5cZtnBnGCYEeI2ZnO+8qBFFAR3AlZm1USJq+51SuAqLh7lvuazWq1ynZ1zjqVqcmElK2WFZVpcQws5kMtcNDg7+ejwe3xmLxS77/PJPQ8dHKegYaTWALwDoA9Bk/qwYryEMcZPfa7CAfjV9ALaZYzqG3LxWA+gCEMdwl4aiwx3oDMMUAgvoDMMw/uMI6F7Oi2oiCIhekqCUA86JELX3OVCICERU5YjkRJQ7V6/GdXrWJHuSUspjOcmeTiHEVJsL5tw7nQC2Qgvpt0AH258DsBHAPuikUOtkvMAxCMrCnQV07/jR4WADOGSOCgBLANxu/u63oAs9mqDtDIu6NgvoDMOEAC/dkVGNHYBoCuhRjJmiuOfAMYXF9SMJ5CZuGqmw+LJlWUeNWJ60LCsphOgTYsp9LB0b7J3mmAnt5rUKwB3QRZ+7oDt/Cy1ADkMHeljE7VKOmQDgEoBfYsjNyxHSVwKYAR0z7YcW3YsGz0BnGKYQWEBnGIbxH7Zwz09Uk2BRe785EVQkiMiybbvOsVofIdlT5jq9zyWSn8yZq9db4s4AWWjBfB90R/oqaGv3tdAdv7uhreC8WiMGkYDgGejRWmcAOmnZC+BJ6Lnpi6GTQ73Q9+5eDHWoTwgW0BmGCQFs4Z6fqMUPQHTf76jtt+iYwuJK27YbXTbrV9y3MLyw2HbFTMddhcVJU1g8EZcrP/H6HNsG4DUAb0HPol4NbfP+MPTz6G4A5zxeKwwW62EYrQWEJ2YiaOeE0wBuANBu/uwBAA8COAYdMx3CxBznAOgOdBbQGYbxCgvoDMMw/sMW7vmJalIEiOaeGQ8opRJ5kj21GLp3FIBOk+Q5LYRocgnkSSllOoCXG7QIP9J6Sejq9c0AFkAnhR6EDrqboZNCJ0f52aAI6vNeyskshyCTQUHMDHfe+w+hE5dzoEX0lQA2ALgInRTaB6B7vIuwgM4wTAjwbOGO6Dp3RTV+iOq+mTwQkaWUqrNt+6qYaYTC4n6X1fpJV7zUEWBh8WQXL2ehu84PQM9qvxW6CHkN9PPoLuhn0oHJeoEIT8wUxBphi5mctS5Bx0wVAJZCx02/AiAN7ea1FxNw8zId6BwzMQzjCRbQGYZh/EfBw4OdCdiimggCopcUiWQCLILzC0eEiKRSqjbPXL2E6/SUK9lzNifZ0z3JXeTFDtrz3R9e7h0F4Ig5qqAFytUAlkOL7LsB7AHQM8r1S7nTIUwd6H7ZBOYSZNIJGEo8nTPH6wAWQieFPgng09CFHk3QyaHBQhZhAZ1hmBAwZkeY67knis+UUYwfovh+T7bIOqVQSiXcMVPOeKo6jFxYfEYIsdflvtUppSzouarIFPv+Het6Xp5xO6GLj9+FLkBeBV18fD90p+9uAMdHuI7fHehhsHAPqks/LB3ouWs5MdMAgB3maMDQvPSV0PG84+bVVsgCPAOdYZhCYAGdYRjGf7x2oEcxIQKYB/EICqtRS4xEbb8gIiubzc50C+ROwge64t8pmCEAXUYkb7UsqznHat2rBXlYGOte8Xov9UF3/H4IYD50UuhuAPcCOAqdFGpBcNX0QGkngoJawyFMySD3Z92NAnDYHOUAboFOCH0WwGeg788maNvCvPepUspZh5NBDMOUMl5jpijGDhBCUAT3HbkYwhCp99k9nsolkDfmKSxOSinP58RM3UKIIJ/rC2YKzkoH9GfsqDkqoZ9FVwH4EoAuDBUgd03C6/KLMMxxd9YJU8wEjF5InYQu9ngX2s3LuU/vgHZPaIIW1EcqlB+GiZs4ZmIYxhMsoDMMw/iP16r5SFq4R7yLhPdcwhCRUEpVj5TsAVBp2/Y9tm3fY05Pu5I9h6SUSSFE0rKspJSySwjBAdwQo90jE7l3nJlqrwFYBh1sPwEtsjdBJ4ZKvUM8LAJ6UAknZ60gO9DzrTUInZzcAz2mYTl0YugpAP0A9kPfq+dH+mHTSQFwMohhmNLGSzzEFu7RJKr7DgVKqTJXF7ljt95IRLMAJAYGBv43cyoB6HYXFjvzyC3L6ohgYfF4Ge/npR/AVnPMhXbyuhPaKekYtMW7Y+/udwe6n5R63OdeJ0wxk9e13G5eC6Bjpvug3RNOYMjNa8RxdrZtcwc6wzCeYQGdYRjGfzxZuJtzLJ9fy1QkqgJ6FDtISg4iitm23TBSNwQR1WP4s1SPSfa0E9E1Usoj8Xj8I8uyOoQQ/VO046AYlFJn0CCAneaYAZ0UcirXB6A7K+IAMj6sHYZETZBrhM0qHvDudtANYIs5roW2KlwOYB2Ay9BWhXuhrTcBAJlMptA1GIZhpiJeEtpRjR2ACAroES22LqVnawDDCotHG09V5To9I4ToEEIkAbQT0ZyysrKfSCk7LMsKe2FxkO/tRNc6a47XoOdQrwbwOIZGDDVCP5cWG7ZwL2ydMMVMQGHz1m0Md/NaAh03PQpgE/QogiboUQRXrmkEdI6ZGIbxBAvoDMMw/uPZwj2igmoUkyJACSZGJsiU3C8RgYiqHJGciHLn6tW4Ts+6usiPOd0QJtnTKYS4Mrezt7d3kZSyLR6Pnw1+V4Hix+e2WBbuY3EROiH0JoDFADZCi5XfhO723QVd2V5M/P4chCFJE8YO9NEs3L1wwRxvArgROil0F/QogtMA9vb29h5SSjnJzDAnnRmGCT9eOtCjGjsAERTQMUVjiACYcu+zKSyuz51D7oqbRiosvmxZ1lEhRIfpIk8KIfqcwuJUKnVHNpu9tqys7OikbCo4/Ho/g/h8pKHdunYDuAbAPdCC+lPQAvtu6NhpxG7fCVDKcVPYBPSpYOE+FoMYuk/rMOTmtQzadW5/X1/f/oqKirPcgc4wTCGwgM4wDOM/TmUjW7iPTJSTYLznACAiyz1Xb4RkT5nr9D6XSH4yZ65er6sLhvGHfPeHX/eODeAAdIBtATgD3ZW+Glpk3w3d7ds/wXXYwt37GghgHWetILoPirEngrbOPAbgF9BFHyv6+/s/s2HDhk0rVqzovfnmm9HS0mIByOa7EMMwzBTGBlu45yOKArpDlPY9KfGGKSyutG270WWz7p5F7i4stl0x03FXYXHSFBb74ebEDDHW58GvZ/Z2ADugBfRXANwE4BEAD0GL6LuhY6mJEAZXrTAK6EF1bBdjT10APjDHLBg3r2eeeeb2V155xb755pt7q6ur35vgGgzDRAQW0BmGYfzH6UD3cl5UE0FAtJIiQLQTYEVHKVXh6iLPTfbUYriFcqdJ8pwWQjS5BPKklLLYlfOhhogwCdb0fs7bSwPYDOBd6G7f1dCz1D4NbQG3G9oCbjyvIQirwDAkaYIU0INap9j26mkYG/eBgYGGp5566oE333xzYUtLCwC0CiF+AuBZAFuJiIt+GIYpJcb8zoqopbdDFOOHKL/fRYeILKVUnW3b7jnkoxUW9ztW66aw+Ir7FhcWRxrns3gUwMfQsfatGBqNdQnayWsvdOfveK9fynFTGAX0qWjh7oVWc7y5fPny248ePbrunXfeqU+lUn8ohFgP4F8BPEdEnfkvwzBMVGEBnWEYxn88W7h7OCeMRDUpErWEw4T2S0RSKVWbZ65ewnV6ypXsOZuT7OnmZE/RCTKYDgp3t28ldHf6KgBfgq5o3w1gj/nfU4WgujXCIqAH3enuC9OmTUv+8R//8U++9KUvJdavX/9NIvofAH4VwNehuy7u9mtthmEYH/BadIwIj76K2r4j99xuilPH/T4rpRJKqQbbthtdhcVOzFSHkQuLzwgh9ua4b3FhcfEJY9zUDeA9AO8DuAE6ZvoUdAFyC7SYfgxT67PMAvrUXMfPtdQjjzyy9ZFHHtm6cePGTXv27DkIYAaA7wL4v4UQDxHRZh/WZRimxGEBnWEYxn/Ywj0Pru7VyO0dvOdhKKXKHHE8p5O8AUA9hs8R7jK2ga2WZTXnJHtSQWyEARCue3i0YL0fwFZzzIVOCt0B4JPQyaBd0MmhseaoBZF4CEOSJowC+nhn+Xkmk8lIADYR/bEQ4lsAPgE9p3LcCCHuA/Bl6Pt9DoAktG3nnxPRrgm+ZIZhmJHw0nUWaQv3iBYOAOF65hyLvM8MRCRMYXGjU1xsBPLGUQqLHav1cznOW91CiKBsmSPNJH1u/XTtGun6BO3UdRxABbRt9mrows5u6OLj3QDG6vQN6hmdBfSptw4QQNxkGiq2EdE/CCFmA3gSOsaZyDU5bmKYkMICOsMwjP8U0oEexUQQEM3igah1kBAAKKUqM5nMdaMkeypd56ddyZ5DUsqkECJpWVZSStnJyZ5IM5ndC2fN8Tr07L9VAB4HMACgCTopdDHPz5eyFaFDWIRtd0GO3/g+NzCbzQrne5GIFPQogonydWgR/m8BHAAwHcA3AXwkhHiAuzQYhvEBzx3oiNZztENUY6bIOQ4QkcxmszNdMVNjnsLibndhsTOP3LKsDi4sZiaJAQDbzDEbWkhfD13geQK6APkQgGyeawQhcPtNWAR030VtF77vSSkFmOJ3IjoP4G+KcFmOmxgmpLCAzjAM4z+FzECPVGLARRT3PpUszIoGEcWcWeTuOeRKqekAkMlknnSd3mOSPe2WZR12J3uEEP2TMFubmRqM9tnw+4YoJFhPQ4vlu6GD49XQNu/rAZyDTgrtN+e5r+8nPAN9aq7jrBVEB3qxRfrfIaJ29x8IIV6Hnnn5JyiOSM8wDOOGBfT8cMwUEkwXefUo46lmAbBSqdRN5vSMazxVi/lvh4mbuoQQY7kgMeElX9wUdAf6aJw3x+sAlkDHTZ8HkIKek74LQNsI1/cb7kCfeus4a/laeKyUkhjbPa5QOG5imJDCAjrDMIz/eLJwF0KoqFXWu4icHeFEZ9tNFkQEIqpyRHJ3N4TpiKhxnZ51dZGfVko1xGKxN2Ox2GHLsjqFEPkqzsNAyb2/46SYwbRA6f3eLgF4A8DbAG6CTgptAvAQdPX5LgBnENysPT8Jo4AehJuF710btm2LYifQc5NA5s/6hBDNAOYVcy2GYRiDwtjfl5G2cEfpPScVi5LbtyksrnePp3K5b9VjeE7WKSy+TEQVQoh0WVnZK5ZlJYUQfVxYHBqK9TxYijdEBtqxqwnANGgnr1sBrIMW2HcD2Idgio4BFtALXScoB8BA4iYUWUDnuIlhwgsL6AzDMP7DFu5jE8Vk0JTdMxFZtm3XjZLsaQAQd53e5xLJT+TM1es186WQzWanpVKplZZlnY3FYpcmZ2eBEspumREI8h72OxEx0QSEDeCgOWqhE0JOYugS9DxKv5+9w5Ck4Q70cZDNZgNJbAkhaqGLRN7yey2GYSLJmM/HzrPlWOeFlCkbP/jFVH6/RygsHtZJjuGFxbYrZjrmOG+ZLvKku7C4v7//iwAQj8fPBr2nSYBjptJbrxjP0JehnyV/CWAR9LPlRgAPArgwwWt7pdQLm511QhHLBLmWUorjJoZhPMMCOsMwjP946aQAIpgQcRHF4oFJTRYopSryJHtqMbw7s9MkeU4JIfbkiOTp0VcZkaje44x3wpBI6wbwHoD3AVwPHTgvA3AbdDJ1N7SdW7G794MSnYNYI2wCut8z0KUzA91n/huASgB/FcBaDMNEDy9xUyRnYgNXxOSo7XtSnwuJyFJK1dm2fVXMZOKmMtfp/S6r9ZOueKnDXVjsZVlE731mik8pxFQKQIs5aqBHYm2Avv9/FzpmagLQW8Q1w9SBHtRs8iAFdMB/C/eid6CPAsdNDBMCWEBnGIbxH68d6MrDOWElikkCX/dMRFIpVTvKXL0G6G5Yh5Qr2XPWlehJSim7C0j2MEwQTNUO9JEgACfMMQ+6E70RwFPQIvse6MRQZxHW4hnoheEUbQXx/eZ7ciuTyfieCBJC/AWALwL4XSLa4+daDMNEFi/flWzhHi1870BXSiWUUg22bTcakdwdM9Vh5MLiM0KIva6YaTyFxQxTivj1rN4D4AMAg9CjsM4D+CSATwE4DD0W6ygmLq6GrSA4FLGMax34vZZPM9CHwXETw4QHFtAZhmH8x2vQH8UubIcoJoOACe5ZKVXmiOOuTvIGImqETva4A5AuYxvYallWc06yJzXRjXiARfjwEqSdm9/4uReCTgT9EsBs6K702wF8Alpg3wXgEIDsaBcYAxbQp+Y6zlp+W7j72oEuhHgawJ8A+BMi+u9+rcMwTOTx6toFRDN2iGrMBExg30QklFJ1rsLi3PFUuYXFjtX6uRznre6A3F6A6L7PYcePYt2RCOKZ3e+46UXoz+Zy6LFYX4TuRHcKkDvGee0wdaCHzcI9kN+bHzPQ3XDcxDDhggV0hmEY/2EL97GJ4t7HvCdMsqcmT7Kn0nV62pXsOSilTAohkpZlJaWUnQEme0ZEiCtvb9Te57Djx/s5WfeI3+u6r3/eHK8DWAKdFPo8gBSAvdBietsEru8XYRTQg/huDMLC3bdEkEkCPQ3gO0T0f/mxBsMwjMHLdyUL6NHC0/OAq7D4SkGxEzsBqMfwwuJu477VallWszOP3LKsjoAKi8eCC4/DR5g+t0HGTCkAH5vjWugC5NsA3AXgFHTM1IzCCpBZQJ/a6wA+x01mBAzHTQzDeIIFdIZhGP9xLNzH6i5XUZzlZ4hqMkgQUcyZRe4ke3Ks1i3Xz/SYZE+7ZVmHTbKnw7KspBCi3yVST0U4EcSMiPneqyWiegDXCyFSRGRh9KDW724Hvxgp8ZCBnuvXBGAatJB+K4B1AFqhk0L7oK0MvRCGJE0YO9B9tz00HehFTwQJIb4NnQT6cyL6y2Jfn2EYJocxvytdRaFRdO6KaswEIhK2bdfkum+5YqYq189kXOOpWpyYyYjkXX78e+kDUXufGQ8QUbm552cKIa4homrosVCT8nJ8uu5o8cYFAK8AeAPALdBi+ucAbISOl3ZBx09eYQG9sHWCKDouaQt3jpsYJpywgM4wDOM/zoMmW7iPTmiTQUQEIqoyIvmVuXpKqesAVPT19X3LdXrW1UV+LCfZ0ymEGK+1M8NMKkYQrzcieS0R1RBRFYBKIkpg6POfsCxrkIgeI6JzSqmDAC6bvwuy28Ev8iUDLgN4C9rifRF0UmgjgAehOyt2QXdajAZbuE/NdZy1grAiLGpiSwjxDQB/BuBVAK8KIW53/z0RbSvmegzDMOAO9LEIbcwEACMUFjcopRoAIJvNPpLNZt2xco+Jmy5blnXE1UWeFEL0TfHCYoYZESICgCpTFFJHRDUAaoioAkAFEcXNeVJKGRdC3EdESSI6QkTu+eB+PntOdsyUhRbM9wFowFAB8lpokX03tKPXaG4SPAN9aq8Dv9dSSgFFFtA5bmKY8MICOsMwjP84HehezotqpF/Seyciy7btuhG6IRyr9bjr9D6T7Bkgopp4PP6ya65erxAi7J3aJfs+M6PidAbFATTkiOSV0Mmecpj3nogUgAFzXATQA6ALQCeALiFEHYClQog5QojriaiLiI4SUbt7PT/34hNe730FoMUcNQBWQovpK6Bn/e2Gnv3XO8L1g/j+CIu1etACut8W7n50UmyC/v08ZI5crBH+jGEYZiLwDPT8lHrM5C4sdgTyK+5bV/R0JwAAIABJREFU0M89DrYQIgkgCQBSyuZYLLbPJZKHubCYjJDKhA/Kcd+qJaJa00leYYRyy3XyILQQ3AstDndDx01JIURWCLGIiBYCWEdEt5oC5Oag9uLTdQv5jktCFx9vBrAAOmZ6EMAD0AXIuwGcxPDXGmQMEETcFNQ4qtBYuPvUgc5xE8OEFBbQGYZhfKampoZ6enq8JDuUh3PCypRPBimlKvIke2ox/GG/UwjRIaU8JYTY4xLIk1LKNACkUqkN2Wz2nvLy8qbJ2lPAcBaoxDGJvIqcjohGIpqZTqd/VbhafYgoa4pEBqATG26RPK/NoCkw+YCILCHEIqXUTUKIO5RScfMaan3aYhAd7oV+DnoAfGCO66CTQvcAuA/AYeik0BEE9+9HmDrQA7EIdK1Vch3oRHRvMa/HMAzjAS/flQrQHZg+v5apyJSPmVyFxcPiJVfcVOY6vd9ltX7SxEoduYXFvb29T1uWdaKsrOzw5OxqUpjS73ORCd1eXe5bddAi+fUAkMlkHpdSpoQQcuhUSkGL5J0AzkDHTF3QMdSohSIm9DoohDhIRNOIaKkQ4kYp5VKlVJ25Rr6xWONlKsZMCjomOgI9zsEpQF4O/Xt0CpB7wDPQx4PvsYxrHfi9lili4biJYRhPsIDOMAwTDGOKGyZBEMVEEDAFkkFEJJVStXnm6iVcp6dcyZ6zOcme7gK6yEOXLBgLE6wwUxQjUNfs379/7osvvnhTd3d3zbJly+xHH300VVNTU05EMde5aSlll23bc9Lp9EIhxAUp5V4p5V4hRJ/XzhnHqtAkWRuJqBFamG8A0AigwnV6Vkq5CsB1piv9CPIklsbBVC70OGWOVwEsg04KPQndkbIHQDnCkaRhC/dxYNu2KJGZrgzDMPkYM6Htes6O4jPlpMdMAKCUSiilGmzbvjKeynHfwsiFxUkp5RkhRNNIhcUemfR9M4wbl/tW3fe///2bN2zYYC9evFhiZPetjBCiL5vNzgMwIKVskVLulFKeRwFCHhHFXE53zuHETPVwPd8KIQallI8T0Vml1AEYN4ciMlXjpj4AH5pjPnTMdDeAewEchbZ+B8IjoIexA70kZ6AzDBNOWEBnGIYJBi/dgVMiITJJUBDCqlKqLFcgN/9tBFCH4RWvXaYTttWyrAM5yZ7R5mkVQpTf7ymJUgovvPDCnH//93+/KZVKxdesWXP2t3/7tw/X1dVNmkVkOp0Wu3btqrtw4UJFQ0NDes2aNZ3V1dUTCvbMZy13HvkV28CXX365+r9+5zvTa3t75RyAvvf88/jxP/9z79//0z/tuOGGG9ox1BGRtiwLUsq4bdvLlFJrbNt+wLbtDVLKXebodtY09oSNrgSrI5bndiP1AOgQQlwUQrQA6JBSdgghOqDF8gVEtAjAGiJa6bIqvIyJMRW7KUYiBWCHOa6FTgqthS7yyUJbvR8EkCnCWrmEUUAPKulUihbuDMMwQcMW7vkJJH4gIqGUcrrIr8RLeQqLk0Yk35cTM3ULIYrx71/U4qapKkwOo7+/X/7RH/3RbV/72tcOLFu2LHe00KRw/PjxinQ6LRcvXtw30WvluG/VElEt9DzySgAJI5Kjqamp7G/+5nvXLl++ufeHP/z+ISnlVe5bQgjE43EopWbZtr1WKbVcKbVSCOEI6cec4iAiKh9BJHdiJrcTVwY6ZuqA7kTvcB3d0LHWLa6xWEkiOmYKkCfyzFgqMRMAnDaHuwD5V8zfrYAeJzbRGHIkgvq+4hno40ApJcBxE8MwHmEBnWEYJhi8JA7Ywn2iF9HJnpo8yZ5K1+lpk+zpkFIelFImhRBJy7KSUsrOIiV78r5cn68/1Sh4v+fPny/ftm1bY39/f2zu3Ln9GzZs6CgrK/Pt9/b1r33tzg9+8Yv1C7JZqxHAv73//q2/+OlPz/3kpz99fs6cOYN+rTsaFy9eLHv2u9+9yW5puaYhm7WOSak+nD+/87GvfOXQokWL+vP9rOkUb3DZBtYQURV0sieBoWIR5bIN7Ghvb+//P771rTsf6erK/Gl1dU8CoFO2Lb/S0lL9h9/4Rt1zL764JXctIUQmFovtJqIm27YXKaXWKKXuUkrdA90draCt9Jy5XwSdUOoQQpwRQjQ5jg7mGEv4PSyEOGw+00uEEHOFENeZWenHiOgwxt+V7vfnstjXvwDgFQBvAHgcwI0APgdgI3R3xS4ArUVcLyibeCA8Qr2zVslZuDMMw0wCXmMmIJrOXUUTkl2FxVdiJqfAGLqT1V1Y3G2e1Voty2o2MVNHEQuLxyJqcRMwjvf57bffnn7LLbf0zJ49O5C45Zlnnln8wgtv39Xa2lb7/PP/35tBrJmPdDotfvCDF24aHMxaf/qnv7ensrJyzO8Tx33LxEx1LpE8AaAy130LOmbqB+AUFXf9+Z//57WXL9de8+GHzdbzzz/f/MQTT5wdbT0pZasQ4mUiet+27duIaKlt279q23YawACAOIbnLFIYEslPuWKmDiFEr2uC1khcdo3FusnMSl/rKkCeaFe6n5/LYl97EMBOc8wH8BvQcdPvQtvm7wJwAMUrQA6bhXtQgnMgBc5mBAwL6AzDeIIFdIZhmGDw2oEexUQQUEAyiIhizizyXIHcJHws1+k9JrhstyzrsOmM6LAsKymE6B8j4AyCSX8Bk4CnPb/77rvT3nr22Vsa2tura4TA+7GY2vr22xf+4+/93sFp06YVvbP2pZdemvX+L36x/tsA/ueqqn4pBJqzWeupU6fmfOc731n3/e9///1irzkWz//bv13XuG/frM/Nn5+cWVmZ6R4ctF46caLxpR/+cNHvP/30XsuyyqBF8lqT9HF3RFzpDDK2gf3QCZiL0DPI3fPIrwTd//Iv/3KL6OyM/WFVVU/C/Pl1lqV+Mx5Pf3vfvgWtra2VM2fOrBqlI8JtG2hDi+dxaGvxASHEYSnlDpM4mnDAapJHW4hICiEWmq701US0gojOK6UOQie3PF9yoq9pEq+fBXAJwHQA/wpgFYBboTvT26CTQnuh74GJEMYO9FDMDeQOdIZhQgJ3oOfBdKh6jZmEUqp6lPFUjRgu0mVc46lanJhJSpm0LKtrKowIidgYqIKfGZqbm6ufffbdJStXHrzw+7//u77Piu/v75c/+tFP16dSs8s/+mj/sq1bt+7YsGFDsS3CC+LDDz9sPH5cNGazCbl58+ZrNm7c2AZcuXfqjPtWXY77VgI6f0AABBENQgvZvQDOYyhuSgK4auTAG2+8MWP37mM3JBJ/M5BO/6jse9/7l3WPPfbYWSGEIKLqnHjJbbde7rpMv1nf6S4/K4RoMi54AxP9vZjP75VZ6QCcrnSnANkZi+X1c17KMRMw1HH+EvQz+moAnwXwMID90HHTuQmuEUYBPTQz0LkDnWGYQmABnWEYJhjYwj0/V/ZORCCiKiOSjzRXr9r1c1mXZeCxnGRPpxBi0qy3PRDl9zsvly9fjr/9ox/dvD6ZrNy0YMGlMsui1r6++P+7Z8+cF194oec/fvWrJ4q95s9+9rNF87NZ6zeNeA4AS2Ix+/F0mn74wQdLAAQqoF+8eLGsc//+GQ9ce236munTq1VZWUVVLFbx4PTplc9fvnzj6dOnl82bN+/K/UNEGSFEiogGoJMCbpHcs6Vif39/vAwQtWVlMbIsC9qjPTYjFotV65l7f5DNXvlYuW0Dm13dEB1CiG6dNyIopeYopdYQ0XLbtpcrpVqMkH7CNcd03Bi3CHdX+i1CiHlCiPnj6Er3exZeEB3uSQC/BLAZwALopNCDAB6AtnbfBeDkOF8LC+jjX8vXTgqegc4wTEhgAT0/w+KHEQqLr8RMRix05/x6zLPaZcuyjpi4KWkKi/umQGFxPqIYNxW031dffWtOW9u0up07L1mHDh06VwwL83w888wziy9eHJheXf3jvlTqa4m//uu/Xfv88xsmrQt9cHAw9sEHTTdXVa1qTCQS8ebm5ns/+cn+nrKysnIiqsDQMyy53begY6Zu6JipEwWIakQkv/e9H9yp1C2JadMeTqfT9fLs2d+5ZcuWLf/r7bffXoPhnz/Hfeu8EOKAK2ZKCiHS5noJ27ZXKqXWEtFnstnsBinlDsuymoQQed3HvCKEcHelLzJd6e6xWAfh3dLcr2fooMRaGzo2OgBdCH4rdBHyauiic6cAeTxFDCygj38d+L0Wd6AzDFMILKAzDMMEA1u4uyAiy7bteifZA6DStu1b+vr6FhkBLO46vc8lkp/ImavXWwwBbpIo1dc9Xjzvd8eOHQ1VbW01D99ww+UyyyIAmFVVlVlfWzvw8o4ds1Jf/vKpRCJRVDEqnU5bNQCsnORhnRCUzWZ9e14yHRFOB3md6SavzmazVTOvvXZWQ0MD2dXVeh6eUplqIKP6++2urq5zc+fOvYChjoiUsSD0siYAVJoClSuzyImo4dFHH5329nPPVfwCqHykulq3r9t29qe9vaJ+0aLk9OnT37Msy0n2jGUbCCEELMs6Z1nWOSJ6w7btFUqptbZt/0+2bSellDsty9pdxKRQUgjx4Ti70oP4/g1SoFcAjpijCsBK6ITQl6EThLsB7IFOGo53DT8IKuEUSIeDIQgLd04EMQwTBsZ8vjPPHpFw7sopLG6wbXsRANHX1/cbJmaqcZ1uu8ZTOYXFbpF8KhcWj0XU4qaCaG5urt61q2P2rFlPdV669F7Nq6++NWfx4sW+daE73efAZxCLLbdjsa9nP/ro2753oZOeOV5PRHW57lstLS01PT21NUuX3paRMkbHjh0We/bsSa9du/YihuaRJ5HjvuVhzRgR1bs6yZ2i/sZDhw7Vt7S0ysbG/xNVVTKRSNxht7WtxD/+4z+p22+//S1XzJT0UuQohEjFYrFtRLRNKTVfKbVWKfUppdSnhBDNpgD5dDGKXczrOSSEOJTTlX49EXWO0ZUehpgpd41O6OLjd6ELkFdBFx/fD+AQdNx0vIDXFTYB3Xc3LUMgFu5KKY6bGIbxDAvoDMMwwRA5C3elVIWT7DFd5O5Z5LUY/nBMAKSU8pQQYo9J9nRIKTullFdZpYWISBRM5DDmngcGBqwyIlkRiw0L0qrjcRvpdCyTyYhEIjHaj4+L9evXn/nuW2+t3ZbJxG6Px7MA0EuEF2zbunHp0uMTubapcK7PEcmdeeQVcAWkpiNisK6uLpkqL6dDR46IGbNnXxSDg/1Qym66eLG6DaDGxsad0LPcRltTGHtCpwvpilAObRtY5jq9F0DH0aNHu3/+3HODkPKaP7xwoWpLW5u9sLw89b5S1kfl5enfevzxn5eVlY27+98khbYT0Xal1DyTFLpXKXWvEOKgSQqdKlJSKF9XerfpSm/B1V3pQSRr/Lz+aK+/D8CH5pgPnRS6C8AnARyFTgq1YOxkRZgE9LBZuLMVIcMwYcDrdyWFxdLbFBbXOVbr7tFU5hnG/cyWBgAhRJeU8qQrZir1wmIvhOL99khB7+Orr741p7d3fmLWrKUXAWDnzh/N8bML3ek+Ly//nRQAlJd/YbCv77tVE+1CN0W+VSZmqic9j7zaiOQVRBTHkNV6VggxQEQD2Wz28uuvb6lvb1/QkUg0XMhkRKq9fX7DCy+8071kyZIxZ6ETUZk7ZqLhI6rqXKdmobvIkwBa/uzP/nJ+W5uYVVnZ1t/V9eMsEZDNzo5t3749/uKLL55/8sknz4zn92AKkE9blnWaiF6zbftWpdQa27aX27bdbgqQm4QQEx3N5Kw3Ule6e1b6aF3pfnag+0m+6xN0bHQUeszFSui46UvQRRhOAXKXxzXCIqCHysLdPD9w3MQwjCdYQGcYhgkGL7ZzJdWBTkRSKVU7yly9BgBuhXPA1UV+NifZ093f3/9bUspjFRUVr0/WfoKmkBmGIcFzELRw4cKepqqqwQMdHRVLGxsHAEARYV9HR1XVqlXnampqih7sfOUrXzn+0nPPnfiNQ4dufDSdLmsUQr2slHWmpqb3H7/5zW1j/bxJ6NQbu8xaAGVKqRWZTGYmdOJT6NOIoC0DB6BnVjsdEZ3mvwoA4vE45q9cOeO9lpZbug8ciN9QW1t+ob+/bGcmY8176KGjs2bNGiQiaTowrnREYEgkb8DItoHnhBD7XHbrSSFE+vDhw1X/8O1vf2LlxYuxf7ruusPPlZVNe+XChZmd2Wx8zk03Hf/GV7+6bbxJoFxMUuiMZVlnTFJopelK/3Xbti+5kkITnvln1svtSl8IYBURLc/pSp9MgTvI6582x2sAlkJ3pT8BLbI3QSeGLk1wjYkQtIDua4eDay2e5ccwDDM2Xp+PSypuUkolbNtudBUWu8dT5RYWd5qY6YwQosntvpXJZJZkMpnPVlRU/DTkYnkuUYubAI/7dbrPp09/uEcIicbGpf2HD8+d6VcXeiqVkj/60U/XZ7PzYlJujmcymwEASl0vPvpoy7KPPvro4/Xr13eO9vNGJK81MVMdgOsBxNPp9BeEEBLD55GnoWOmPgAXMHwe+RX3rS1btkzbubNnZSJR13vhwkEBoKK8vCFz+nRZ43vvvXfNQw891EZEFS73rWEFxtBOTQ6DGBpRdSZnRFWPU+irlMKhQ0e/DmRt2/4WysoqsgAQjyMrBFFra6v7muNGCNEfi8U+JKKtSqnrTQHyA0qpTwsh9kspd0opzwbQld5FREdMV3oYYiZ4WKMfwFZzzIWOme6ELkA+Bm3x3oL8XfosoBe+Dvxeiy3cGYYpBBbQGYZhgsGLZdeUSwwopcoccTwn4dMIHfC6K0S7TIDZalnWgRyr9bGqo6fc3gMgSkmvgli2bFnv5ttuO/vCu+/ecKqnJ9FYXp452N1deaihoW/Txo1FEXFzSSQS6rmXXnrp6aefXvPyu+/ekk6lym9cuvTEf/9P/2nHXXfd1QEARFThJHtMR4RjG1hBRGVwdURIKXuUUtMGBwfrTIfQHinlaWjB3NN7/+CDD14sKysTzdu2LWju6rqm4oYbylbdfjvWrFnziXQ63QA9q835DCoAjkXgCSHETleyp3Ms28BXXnll7qyLF+v/l+uuayuTEmvq68/+5c03n3361KmZ8YcfPlAs8TwXIcRALBb7iIg+ciWF7s+xKjzjd1e6UqrLvJd+fw9NpQ73QejEzy4AM6CTQqsA3AHgjPnzZphuNxdhE9DDMgNd+r0GwzBMADjOVGMxpZy7iEgopZwu8ivx0iiFxSmX1fq5nJipawxh3D3/PUqxRJT2WhCvvvrWnMuX62tmzhzounjx42oAiMVm2jt37vSlC727uztmWbAbGs51Av/V/VcDsVh59sKFCxVE1A0dLzlxU41x36okogSGP7NKANl0Or1YCHFZSnlQSrnfCNgZL6+ptbW1sqZmMBWLvV1eVhavKSuLV8bjscr582sqAXlzOp2uxvDPYB903NQhhDgGI5hLKZMA+r3EHZs3b54+e/Zd7bNmPdhuWa+qv/iLL7+zbt26UQsHJooQgizLOmFZ1gmlVLVSyulKv9W27TYzK32fEGJUh7IC18vtSl8A05WulFKmeMHPz+VUENDdnDWHuwD5cegCjybouGmkMWFhEtCDKjqG32uZwmOOmxiG8QQL6AzDMMEwJS3cTbKnJk+yp9J1etqV7DkopUwKIZKWZSWllJ1GnBr3S0H0BHSA9zwqX/mt3zr6szlz+rZs3Tor09ubaFiy5NSjDz54dvXq1YXMay6I+vr67DPPPNNMROcwJJKvzWazCeiETwxDInkGOmDuhw6Wu6G7yDsB9MdiMSil6m3bXquUWq2UWgjgmGVZH0spD7uTo8Y28MpMPXdHxJ133ll75513IpPJIBaLZR1RHLo7wN0R0T2Rz2D76dN1Ky3LLpNDX0ExKbE4Hs9uO326frzX9YrpSj9pWdZJ0rM+naTQCtu2L5qk0F4nKdTa2lr+y1/+cmbbsWP1ZdXV6eVr1rTffffdl6Uc/St0hI79GlN9fg2AG/XLENcLIe5RSjVj9Fnp495mka83EuNNoFyETgi9CWAxdFLoswAeBrAfOil0DsEkaYKaTR4qC3fbtrkDnWGYMOA1Jgg8djCFxU7M1OjuJMfwokYC0G2ez1oty2o2MVOHx8LifLgF9KgRpT17fmbo6OhKzJmTTgLPXfmzhgb0W5ay29raEsUW0GfMmJHes2f7v+S6b7lE8juz2Wz5lY1ooXXAHO3Icd+SUqp4PG7Ztr1YKbXOtu27bNteaTqrd0ope13XEuZZflgH+ec+97nGTZsyDQDirpfaTUQdZWVlSSllhytmSk5UZFZK4aWXXls4OLgsft11j188efLIzBde+NmCdevW7ZzIdb0ipeyVUn5ARFuUUguUUmuUUhuVUg8IIfZZlrVDStnq/pldu3bV7du3v/7Xfu3Lp7ysQUQJ8z03DdrhrMwUG0+H+T0LIR4GcISIDqO4z6BBfdbH82yehnbr2g0dQ66CtnlfDy2w74aOnbgDfXwEZeHOHegMw3iGBXSGYZhg8CKgK0AHmcXotnQgopgzi3yUuXqW6/QeE1i2W5Z12NgHdliWlRRCeKrGHu/LRLSSIkD0OikK2m8ikVBPPPHEueT997c9++yzNzQ3N19z5tlnq9rb2089+OCDF8f9IvS8q1pz7zsdEdXQXeQVGLoPBRENQid7egGcx3DbwNyu3KuQUnZKKd9SSm02XQLrbNt+0rbtlLmGgk645rMN7BBCdMRisaTbNrDYVE+b1n/Wti1FBOlYExLhbCYTq5o2zZf5iaMhhOiLxWJbiOhDpdSNJin0sFLqfiHE/vPnz+/717/7u2umnz49bVV5ebYzk5Fbd+6ce/LEiaNf/rVfO01E9bmFCC5b+9yO/Q4AB81/B4UQNUKI2UKIuUTUQ6PPSh/X1jA1LNzzYQM4YI466KTQKmhB/aK5fu+oP10cwtqB7reALlGc+5RhGGYy8dqBXnQLd1NYXD3KeKpGDC8szjhinJSyxQh0SSll0rKsrrGcfybyMs1/oxg3RW3Pnvb7V3/17Salhmpot2/fXv/OO+/M/oM/+IPmWGxiKV8iSrhipnzuW7aZR94PLYw7MVMnPLhvCSHsWCx2AMAB27ZnKaXuVErdrZS6B0AHdDxWgeHP8mSu3yGlPJVIJHbnjKjy7Zlo8+bN0w8eTM2ZNu3hLm2bv7F7z56/n799+/Zjfnah52K60o9alnXUNEWsVkqtzmazqwGcN2Ox9tm2nX3xxTduPH2aGpYt29m5Zs2aLlPYUOnY2ufETI3Qv2+Hfuj4tF0I0QIdQ5ULIWYAWENEK8xYrGaMPCu94K1harl2jUY7gDcAvA3gJuh46READwE4ac5hAb3wdeD3WiygMwxTCCygMwzDBIPXDnSgwIdTIoLp2LyqG8IEQNWu07OuWeTHHKsyk+zp9DPQHGsbiF5SRPueFblgogTwvNlTp04lfv9rX3sAx4/PXqmUaAfE37355vKPPvro46effnrfaD9HRBaGd0TUmo6ICmMb6D7XmUfegavnkXv+PJjPYU1uR4RLuHWvGwMwC/oeaBNCbJdSHpNSdgAYmIz74d5PfercD7dsWfDj8+cbN82Y0SmEwCsXL9Ydqq4e+MJ9950L/AXhSlLomGVZx0xSaJVSas3B/ftXLVVq4D/ccceFBJCiWKziYEdH7bunTq1ubW1NTJ8+3fkF2hiyZzzi6jzpEEJ0jdaxT3pW+gIiWoShWemtPnWlF5NiJza6AGwG8C50h/5qAEug7d4fh+5KP17kNYHghO2gOt2BYCzcuQOdYZgwUEgHesHOXTmFxQ2u57YG89zozpH1mLjpsmVZR1wCeVII0TdJz+9RFtCjREH7dRyYlFL4L//l79cdPpycu3z58subNm1qy7uIFlGr6eoRVRUAEqbL3Dk3r/uWuZa3zRHFc523XDFTHYbbONcDmAZtrd5s5n5fzPcs7yfu7vNZs24ZBIDa2lsHOjoW1QbZhZ6LlLJHSvkuEb2vlFpoxmI9opR6sLm5+WwyWRuLx6fH9+8/fP3y5ctj0CJ5uesSvRgukne4OvdH7dg3BRZLhBBzhRDXkZ6VfpT0rPSpWthZ7FjDBnDQHLUAbgVwm/m7pwB8DGAv9OiAYhNkZ3iQArqvn20W0BmGKQQW0BmGYYLBywOg80B61cMcEVm2bdePkuzJtSvrc4nkJ3Lm6vWOMVdvsoisgI7ozTD0zN/97d8urzt6dM5fT5vWObeszFZE+HFnZ813X3hh7faHH2677bbbBBHVYUgkd3dEALojQmHINrANQyJ5EjrxU0iximMb6CR6GjBcJB9mGwidhGgVQhzIsVtPE1GlbdurlFJriehe27ZvJqKPLcvaj0lINqxfvz55+td/fedrP/nJ8ndaW6cBwGBDQ8+djz22/xOf+EQxOgkKgohiuUk1ImpUSlHroUO4c+bMinhDww02AKFU+qZYrPvD/fuTzc3Nx+69994TLlv7gj9bJhl3RAhxxNxfS8ys9HlE1G260g/D41xG96VRGt0UuRCAY+b4EnRi6Brzv7sA7IG2K+wq0nph7EAPwsKdE0EMw4QBr9+VI8YOOYXFDaaw2B0z1bhOt13jqY45Arlx35rMwuJ8OAW4ImIFuFGMFQvm5z//+bWHDnXN7+ubU/HDH/54+caNG9vMjeK4bzkxk+O+lYB+RhEA4HLf6oF23+rCUHFxQbbnNGQBfiVecnU3uwv80xhy3zqXEzP1AIBS6gal1DoiWkpEi4hoj2VZHwshAo9R9u3bV3vyZPd0oEWcPPn1Gc6fC0Foaem59tixY5ULFizoD+r1mPi0OqcQYZophsgopcr27Gm5sbZ2ORYsWKyOHPlZ6vjx4ycWLlx4IMfWfkx3tZEwP7uF9Kz0hUS0ELorfSURnRtnV3pQMZMfa3QDeA/AKQC/DuASgE8B+DSAFugC5GNFXDtsHehs4c4wzJSDBXSGYZhgyNuBbpI9FgCk0+mlRFSbk+ypy7lWp0n2nBIF4A3kAAAgAElEQVRC7HGSPcYyelzBzyRDpK21o0gkBPRChUzbtnH4449v/s3qasxpaKhUsVgclhV7bPp06/XTpyv279//K6tWreqBFsmzxjZwADpAd9sGFmQ3TUQWXW0B7rg5jGgbKIQ4JYTY7UpCjJl0FUL0u2zKF9m2fZtS6rNmdt1uMys9MAtAAHjiiSfOfvrTn27bunXrNCLC+vXrL0+fPr1QkdgzpGfpjdR90ggt0jpkMJRY2995+XLDYFdXOp7N9qjq6pmqpmY+lZdPj5WX9wohlJTyrBCiKK/bdLdsJaJtrq70W11d6Qehrc09XQ7+f9b9vr6Cdmv4MYC50PbuGwDcA50M2gWdHJpIQiJoAT2I7qUgLNy5A51hmDAw5neyiZlIKTUtlUrdRlePpypzne4uLD7pipmmcmHxqLhebxTjpqjt2fN+ichSStW/8MLLdwuxtua66x6zz5z585Xbt2+fuWbNGvf1yMRLKRTBfQtAVY4FeIPrWd5tAT6AoWd5p9A1ady3xnRzsCzruGVZx5VSdbZtryWi1dls9nYAx03MdDiobvSbb76596tf3bRjcHDQyv27ioqK7Lx58waKvaYRyWtz4yXX79qd2++C/l2fFULs3b17d2bz5nNz5s17GIlE4+xMZt41TU3NixYsWNBjxk9cKsZrNGMrWoQQLTld6fNzCpC93GNB5UeCEOlfg77/V0C7ef0qdK7CKUCeaLwfpIAeVMwEsIDOMMwUggV0hmGYYFBEJJVSThf5SHP1ygEgk8l8DtrG2Un2nHUlepJSynF1VU5xothVENUE2JX9msRLLQ3ZBtYQUQ2ACtu2EzXV1bNqLEuq6mobSikoZYtsNmtlMun29vZTRLQHupM8RYXbBjoFKiMJt85rzGIcFuAF/TK0Tflhy7IOK6UaXUmhO4QQR6SU29va2s60tbWVNzQ0ZK6//vqiJ2XcNDY2Zj7zmc9cKNb1iKjcdI5fSfSM0n0y4ux3c/Q6ibV4PI7q+fMXbHvzzZuWJBI9NQMDXWhvP7Ktr+/6nvr6WfctXXpPJpNZL4Rosixrh5SyKJbrPnalF5MgEijuNc6a43UAS6HF9Mehk0RN0Ekhr8UFuWsAwQnooUg6cQc6wzAhgQBAKVWR00V+xX0L5llNKbVSKbUcurA4KaU8I4RoynHfKsXC4nxENX6IWqx41bMJaYetBrrafStBROU7duwoP3q0Z8bs2V+0KyuX04kTi61//ucfxW+99dbdUkpHJO9GAc8jRritGUW4zS1W6cHIFuBJIURq/L+KIaSUXVLKt4los23bS5VSt9m2/QXbtruklDssy9rV29s7mM1mRUNDgy8OEolEQm3atKlosZIDEUka7nSW+7t2BPt8RdxJI2QDALLZrHj55XduHRhY3p/NXnv5zBmcSadX1G/b9uL1a9eeXjlv3rz1AE6Z391B989OhJyudKcAeTXpWennlFKHMLljsYL4LnHHGQMAtpljNrSQfjuATwA4AV2AfAjjc6ELWwc6W7gzDDPlYAGdYRgmALLZbGMqlfoihj9Id5ngotWyrANKqWql1PpEIvG3sVisexJf7mQQtaQIEIGuc+BKcFKnlJpp/v/ybDZ7A4BKIirHkG0guWwDu4QQZ65dtCj1yhtvzH/Ati9WGLF6a19f4rhtD37qppt2Axg1eeGyDXSSD+6OiNFsA/e5kg8dQoieIItVTKHMG0T0jm3bywYHB9ft2LbtV8/v2dM3eOFCa3cqdSZx002tjz311MnGxsbJFGqHkfO7npYjkle6Th2p+6TDdJ/0j9V9kk6nxQ9+8IMbt7zxxqK248dnv3bixLX3zZzZUVdZOXiqqqpj4V13fTRjxow+27ZXE9GqbDa7DsBpV1KoKIm0CXSlB2FHGKSA7pCGFst3A5gOnRRaCWA9gHPQSaH95jyva2CEdYpN0AK67x3opOeTMgzDlDKqv7//dqXU511/lnJZrZ+VUiYzmcx9Usr9FRUVr4ewsDgfURbQQ40pBq50CosB1Gez2U9DP0snaGhEFXLdt5RS3f/wD//P4kuXFlQAK9v7+yWkfCzx3nv/e/y11167sHHjxlFnoRvh1nHfaqD8wq3T3XxGCLE3R7gN7BlECGHHYrG9APYqpWbZtr1OKXWPbduffPXVt860tXUc//rXv/xBPB6fUvdNzu96JJHccTq74jgohDg2QhG3J+Hv3Xffnfbaax+uqK3N9gsxNGmsp0e1P//8z7d94xu/A6XUGqXU55VS/UKIPZZl7TSx2YQxr/OwEOKwKX66xRQgX0d6VvpoXemlbOE+1hrnzfE6gCXQcdPnoV0h9kLHTaN+XkdZJxSxjGsd+L0WafdLFtAZhvEEC+gMwzABYFlWRzwe/1gIcdiyrA4p5VXdq+l0+pZ0Or0+hJ0SXoiigO5Q8vsmojiAetMR4XSSV0HP1is350gAdiaTWZrNZpOWZZ0UQhwXQrhtA4d9Jh578sljf/nxxw9//cKFxrvKyjKXslm5WQjMXL/+4EMPPXSBiHJtA6/M2cNw28B+6GRPUghxwvxvz8Jt0AghMrFYbPdzP/7x5cwHH6z65IIFM2YtX76go6/vho+OH///2Xvz+KjKe3/8/TznzJZ9IwskECDsa0jCVpRFFsUFcUOrrcvVXmtrvbe/295WW229/d3WUq2112rdC0WrbAqyLwoYWQKELOzZScg+ySyZmcyc53m+f5wzZDIkJIEkCpz36zUv23BynjlnJjPP5/3+vN+f+o1r1+Y+/MQThy7n3CqRsjXV4XCYhg0bZl22bFnpoEGDLjnT0E/odXCvO4po9N9rKyGkCAH3mhByRe753zz//JTa3bvHLwAQb7F4su328M9rauImLlyYe+999xVnZGTYAMDvTOGcj2KMZXLO7+KcuwNIoV6Zl9gNV3qpEOIU2rvSr8YZ6MFrXOoaGgBsB7ALwEiopNBtAG4GcBwqKXSuG2ugi3V6A/09A72vnRT9Fa2oQ4cOHX0JYTAYSoQQBymljZqL/CL3qs/nu4EQ4r3OxHPg+hXQgWvgmi+RvmUWQligcbSEkAjOeYTH48milFZKklSsNaD66yZ3YPrW5s2bE3Nza6IZG8Wbm1eF+3/udMaE/vOfn0y45ZZbGoOTtwJqpii03VuGjoXbJqKOqPrWCU6U0mpK6WdCiO1nzpyZXlLizSIkMrW4uHh8WlraAUmSCi63iZZzjpdffnns7Nmzz0+dOrVbUdtCHQfW2Yiq4Hvdp0lnx44di3O5DCFGY7YyeXJopSzLF940CQlxLlmWywEUcs7jGGMZ/hQ0qNH4Ryilp3orGl97D30thKBEnZUe6Eo/rzUgB7rSr1UB3Q8f1MSuPACxUJO8JgOYClVgzwVQADWlrat1riUBvT9noOt1kw4dOroFXUDXoUOHjn4AIcRrMpmKoc6I7QzXMyFyPQroV9XrLYQwa0RLpBAiAkB4YGxgwHEswBFRjYB55AaDwcMYm8g5n6YoSjqAJErpQUmSznVUnGdlZdl+9/rre1d/8smkradOJYVGRdG7FyxQbrvttrGMsVmMMUPA4f7YwLqg2EArIaSrwvNbB5vNJlcfPZp4i9FYO8rrLRHnz5tCoqJSZ6amDvvS6bylqqpqUkJCwgFJkk50l8x6/fXXR+xesWLGcJfLkgaI/L17cXDHjtG/Xr5817hx4xxCiLBOHBEXRkxocKLjiMY+u9dHjhyJrPjqq1H/YbF45kVFtQDAXQMG2H5fWRlfUFUVnZ6ebgs8nhDCJEk6IUnSCc55rEYKTVYUZQaAMs2VfqoXowoDXenDNFJoohBiXIArvT/+1r8JB3pHYABOao8IqISQnxhqgEoK5QFo6WQNdHOdK0G/EDQa9Ah3HTp06OgehMFgqAVwGpf+TBNo+xy/nnBV1Q+9iKumVvSnbwWJ5GFQayYzAhzdAelbdqhjcewAmiVJsgGI45xP55yP45wna42ghyilFzWkulwuOSFBtlGaYzYYco2yLBlkWTbJssk4dGhils/n+07A4T60JUKdDBJur9pRcUII99atX1adOpUsIiJCYg4fPh43dOjQOzjnCwkhudqs9KaenHP79u0Jq1cfzDp27GTVypWZOyml/rVkTSSPDa6doKYH+KHgG7rXXq+XHDx4fIjROMPr8500JCZGux555JGyjo6llDZQSrcJIXYxxsZyzjMZY/cyxloIIUclSTpKKb3SWd0ALjQgd+RK989KLxJCXDRfvo/wbRHpGwHsBLAbwAioDciLASyC2oCcC6D8Euv0l7B9LTnQ9bpJhw4d3YYuoOvQoUNH/6A7mzP/JvF6JYOuClKkF/GtIic0F0NYANnjF8nNUJ3khoBjfVDJHjfUTnE72lzkLaKTeeSEEMiyfFgIcZhzPpxzPo1zvkQjNoq0c4VCne8XAyB69OjR0q+ffx5oHxtYQwjJC3BE9GtsYH/AarUaJLfbMDA62g0ARFFa5YaG04mMnbU3NKQ5HI7I8+fP37Xx009vL8nPtxOL5dzcm2/Of+SRR0r95E4gioqKQnZ9+OHUBzk3fn/YMBcxGIzNgPl/qqpGbd64cdjIkSNNaD/H0A71XlcTQo4HOVD6NCWDc46GhgZjaGgoCw0NZQBw+PDhuEi32zR70KALzgSJEMyNjHQerKgYUFtba0pKSupQvNccbNuFELsZY2M455mc83s45y0aoXa0p4RaZ9BIoSJCSBHnPF4IkQ5gBCFkouYQHgjAgL6Zld5fBEpP17AD2AtgH4BUqKTQPAA3QRVIcgEU4WJR4FpyoPdXhHufzPvUoUOHjn4ER/c+L6/H2gHQ7o22p7ie8G2rm2S0zSMPTN/yNxZT7TgONZrZDVUksyOguRiX4AgkSTovSdI6zvl2xliWECJTG090jhBSAXUvGSOEiLnzzjtj7rzzzsCxSR6o+/hKtKVw+ffyzm9b+lZvIC8vL+LEidb4hISpTkky2r/8ckXD6NEF2yZNmjRSc1bPWLXqw9rq6tryX/7y51u7Eq8ZY8a1azdObW1NjamsrI86cOBATGZmpgWqSB4RcGhgQ0IhIaQx6F73+XvX6/USo9HYbp21a9cml5WJxISEx6xW68aIDRu+GHvfffdVhISEdNrQSQhRAqLx47UG5KmKotxACCmilB6mlJ7tbVc651wGMBHAaELIXK3hRAAYgL6Zld6fNUBP1uBQa6PTAMKhjsTyj8ayQk3yyoPazO7HteZA12eg69Ch41sHXUDXoUOHjv4BR9ckDweuS0IEuH5JMAghSH+RGNp7KzA2MCLAEWFBmyMCAY4IJ1QnuQ1tZE+PXMZCCIPWpd8uzg4qASGgCvQTtMNbAdReyby3awFxcXFeJTTUe85uNyWGhl4QW8vtdqODsepNmzYd3/Xhh3OmMRZ5f0RETHFdXezWt98e09zcnPPMM88chkao+e/zyZMnE5ONxsj7Ro8mQpIgAEQw5rvF5VLe2LePu374w33h4eENASL5N9KQsHLlyiFr//nPKfbz5wdwSWKWpKSGjIyMcpPJpPgA4eKchkvShYLarihUyDI3m81dvjc0UqgAQAHnfIBGCmUpijILQLEkSYcppWd6QgqJzue/x6J9tL0HgEOSpFgAdwshajRXek9m3HV5ifj2ONA7ggBQqj0sACZAJYW+CzVB4hhUMb2/BfT+iO/rc9cG51wngnTo0HEtQKAbdYH2XX091g7XqwMd6Odr1tK3oqCOqbpk+hbaGotr0CaQ27T/3e3vfyEE6SQRKnAWeYoQIgXq/sUGoJJSehZtY5OarnRs0tUGzjn27j2U2No6XI6IGNyq/mw03bXrQOiUKVN2EEK+KCkpyVi1ass8QmjCvfcWjRo6dOghSulxaGOqEFQ7nThxIuz8eY7Ro59Cbe0GvmHDVjkjI6OcUnouqEb9RhsS/vCHP4zbvfurUWvXfvTZ1q1bExYuXFhrsVj4+vU7xnL+HWI2p/piY++ynTu3L/6TTz4Z3JkLPRiU0jpK6RYhxE7G2HjNlX4/Y8xBKT2qPezdfZ7i4vnvsWhLO4tCm4mEAfAQQtyU0kVCnZVeIoQ4jYtnpV8u+mvsFXD5+38HgK8AZAMYAjXJaw7UBuQzUMX0IvSvsN1fNRPQ9w50CXrdpEOHjm5CF9B16NCho3/QHYH4eiZErkcBvU9eb60YiNIInwhNJA+BKlJbAtYTWsy6B0ATgAq0ET1N6GGBKoQwCSFiNAIiWrQXycMDDg3s0j/ud0QAcHLO04QQUwEMFup8tEpKaUlvdbl3Bs451q5dm/z1jh1DW+rqIsIHDmy+cdGikiVLllT35bqXQklJSWijwcA+PXs2nnNORsTEuKtbWox7rdbQkKysc1vWr592q9dr/vmgQXZiMBhhNFo+bGgwrt61a1bjQw/NioqKAgDhdrubW1pamh0OR73iclGj1WqVFMVLfD4vhBCS3R7iAOD1eg9JknTBWV5UVBTy4YcfptlsNvO4cePqv/vd75abzeY+fR0++uijlPdfeunmuR6PaSSlbIvdHtdYWTnw6NGj43h4uKORUraCUv6DxESrgVJUe73y505nyKC5cwujo6N79H6llNZTSrdqUYXjNFJoGWPMEeBKtwGXFMljAAS6fVwAGjXnyVkAjcHR9kKISCHEGELIYELIICGEUyOFgmelXw6+7QJ6INwADmmPgVBJoakAbgBQpx3T12ks16IDXSeCdOjQcbWjWwI69Aj3665u6u0mcy0xK1S0H1EVptVL3Unf8tdMHY2judS6VNsPxvibi9EmJEajPU/rT98670+EgupEDeOcZwghRmvP1SFJ0jn/3rWvwTnHiy/+fuKgQfHOJ554oqQ/1rwU9uzZE3vwYF1KQsICp8fTLAFAdPRo19mzp+IOHz6ckJGRIa1YsTKpri5SkSQLWbt2fchPf/ofCzjnCwJO425tbbUajUYrgNJ33lmRWF2dEpuQMLGOMSHv2PFL86xZO7667bbbajp6Dn//+9+H33jjjbVjxoxxdvTvfYHa2lrjunW7Mpqa5Mhnn3126q5dxydt2bLl5IIFC0rKykRibOxdNgAwmQYphNwoNmzY3aULPRiEEJ8sy7kAcjnnSYyxDM75TM75jYSQM5orvZgQInookjdpNdNprVbyO/ft/nMBSBNCpAFIF0JMEOpYrBO4clf6t2EGenchAJRpjy1QG5DTATwA1eggo/2otb5CfzvQ+2OUl1436dCho1vQBXQdOnTo6B/oEe6XxnUnoF9JnJsQwoiLRXI/2WOCdi+12EA/2VMHtZPZFvDoidMW0Dr0A4kejXQKFhLdUItiK4DyoPjvTrv0JUmqF0Ic4JyPZIxN55zfzTlfSCnNkSTpCCHE1YPb1G289dZbaQUffzxlJmNUBrD5zJnBf9qzZ/zu3bvzli9ffjA4Eq8v0dLSIv3+xRcn1x0+PMzc2hpWypjltN2emjVhgi8mJUUedPPNpoSkpO98uWYNXZKaCoSEQDDmI4z5FoeE2FZWV5vy8/NtmZmZ8Zs2bhSHt2/nrvp64ZQk53mXK2xrVZW4PTq6FQC8nGOb0xkSM3lyUXx8/AXx/P333x/69ssvLwxzOkPjCBGH1q8Xa1atqnzrH//4PDk52dNX17565copN3g8pl9FRdl+e/580gLO6U2yrBzinCqKIq9izPymwcBzq6rkeEAUUSpYWtr5X/3oR4WXu6ZGCh0TQpxijI3gnE8WQnxHUZQboTaXELQnJboUybtYz0YIOSDaZqWPhDorfbxom5V+ua70/nJT9Pbfw3ntsR3AWAD+OZ0/ApAP1WHRm059P/pbQO+PGeh6hLsOHTqudnT3s/K6qx00XLcC+mX9UsfpW6FoaywOTt/yoC19y+8kb0LP07fkICHxglCO9kIiB9Cs7SNLCSFHAuLWmy6RvlUrSVIx5zxKi3efoijKDELIKUrpAUppRV+6og8ePBidnV05wmwu95aVlUU+/vjjp4YOHfqNuN6FEHjrrfemnz1LBg8YsNXKWKjJYJBCoqIkS3x8vMXhcM6trq7GoUNnERv7GBfComze/LJyxx1FeWlpaRao7t4wj8dj/cUvftMyYcKI40OHDm0+cKAyNSTkh1bOKbNYJjGbbXL4Rx+tG7948eKa4HFZu3btGvDnP79/844du4vXrPl4a39d+xtvvDGmqSk0Soip2LRp1dTW1hmmAwfOjWls3Bja0hJnZuyDC0+UMadUWUkGrF+/PvnBBx+suJz1KKXVhJDNnPPDnPN0IcRoxtiDjDEFaiPwhfEF6IZIfqm1gmalRwIYq81KT+kFV/rVJKAHwgMgR3skQk3yytQeA6DWTCfQN/XANRPhriiKfw1dQNehQ0e3oAvoOnTo0NE/6DJmMMBle70RIsD1S4IBHVy3X6wOIHsi0d4RYQw4ViGEeIQQLqiOBDtUobxJ+2+3ocUGhnfiiIhB+xnZTqiOCL+Q2CuxgYQQIUnSaUmSTnPOExhj0zjnN2pd7gWSJB2glNZ1fabuoa6uzpi7efOYeynl530+6W+lpUNiGKOJnNNtH310075t29LHpaVVJo0aVXPPsmVnp06d2txbawshJBEUa79x48YU4+nTsf+VlmYYFxsLq8eDfxUXizM+n+eeBx+sGjBgQENRUZHL6XROtrlcHtlkuuB6cfl8ktvjCbNarZt+/5vfjLAUFk58IDExMiE1NTavpsb9T1lu/Z3DIR/zeExJksQOK4qhOCbG9qNHH833n6OystL89iuvLLytpSXk1xERjnBKRYHPJz9dXDzk+V/9avp7H3zwZW9dfyBcLhdtqqyMn2c2e752OsMkRZEfkSRfOKU4IwQdaDAoDkLwgST5Eu+/f7/T6TTNGjaseenSpZWRkZHdIgmEECbOeWzA+zpWdO4k90CNGTcDcBFCjkuSdJBS2tgb16uRRsWEkGLN8eQnhZKFEI4AUqin8+b7K/a8L+CDOtdPAXAP1Ej38VCd6dVQSaFCqK9Nb6A/BfT+iHDXZ6Dr0KHjWkB3HejXZYR7gOh03V07OrnmDtK3/PPIQ4QQ5oDfE0II/zxyf/pWoEje0/QtY/A+PsBFHhlwqAJtDjkAv5BoDRASL1soopQ2U0p3CCH2MMYmcs6nMcYeZYzVUEoPSJJU2NsjsDjnWLNmQ1pLS5q5ru5odF7e/lFVVdWR77zz1p6f/ewXWTfdNKeiM5f25UIIAdFxtH1MWVlZrCwnGwcMUJCVFRE/e/bsFs55kxCiXgjRHBUVVffHPy5PLCkxjY6Lu7WREIq6uhUxr776V/tbb72xSQhBOOcjs7OzZ1dXh4wk5PywEyfO1rS0xIQKcdzhdh83q88hFKdOOVO2bNmSeOutt7a7vjfffDujpSUuND//3IidO3cemT9/fl/M7W6H2tpa4+bN2ZMk6XuKLKcrtbXb4i2Wea6Wlj2hLleJeenSYYdUaqANhCRj5MiRXcauB6QktBtP1YmTvBnq31i49t9ySukRSmkhpbRX9r5aA/J+rQG5N13p/SGg9xVqAGwGMApqI7IZwFIAiwEUQK2bejNNr78E9D6PcPf5fLqArkOHjh5BF9B16NCho3/AoEe4dwqNDLqurltoKjnnPBmqKB4Odbaefx65HHCsFyrZ44LqxAyMDXRrp+ruuoGxgRcc5F3EBlYRQgqCHBE9FfZ6DEppLaV0g1Bnr2VwzrMURUkHUKYJ6WeuxMkPAPn5+ZFGmy0kPiTE88KpU2mPcU6eMRi8H/h8hijGDLGNjXGpnEvVVVXxfz52LPWJ5577Yt68eQ3dPb/mQAkm12KhirbtyDVFUax5X35pWEipfTIhNlJT40ny+TzflyTlpwUF0fv3799z9913V40aNQrGxMTolWfODB4fF+eKkCTRyjneam6O4HFxTdHR0a3W3Nz4Z0NCyid6vR7u88WMGjo0UUhS1DuNjWx1U1Orz2Zjcmxs491LlhwLvJ5//etfQy1OZ+gvw8Md4RrpMcFgUB42GLzLc3JGOxyOfeHh4b1ebJrNZi5bLK1VNltYmKJIsQDCKYVXCHgAmCgVww0Gr7G11fTUU0+dDgsL65Bs7EAkD7zfwSJ5uwYQSqnfFXHBZcQYG8g5zxRCTFYUJZMQclqLKiy50veeHxp5GuhKH4E2V3qNEOKkEKI7ROTVFOHe1RoA8AWAHQBGQHVYLAawCKqz4iiA8itcp19m7Gnojwh3PYpQhw4d1wI49Aj3S+F6rRcFgFDO+Ui0ieT+EVXB6VsetNVMDqg1UzN6mL6lnc8SkL4VPKIqNODQVrSNqDoXUDP5Z2T36R6AEOKVZfmwEOII53wY53w65/xOzvkCQshhSZIOU0p7JVr84MGD0fn5zYOjox+xnz+/PcHjSbR89dWxSa+99lp1dnbd6HPn1sTMnz9/e09HP4nO57931MjtgHq/q9eu3WCrrIyPMhjGudasWWNfsGDBloSEhAt16smTJ8O2bTuSZTL9p4dSNVQqJOS7nkOH/v9ROTk5BVlZWc0ul6vo3XfXDWtsnO1raTkVlZzsSJo7N1ZqadlFjx49JsLDwxoTE2OslMrc5/O1+9zZtWvXgLy8ihEhIX9saW191/Tmm29nzJ8/v89d6H73eVTUg7b6+hUxwAyiKANks/lmT1XVm/EPPPDA9nHjxnX6mncikvvrpo7i1q0BTnJ/DXXBSS6EMGtNHJmc87s453MopUckSTrWWylyl3Cl+xuQi4UQZ9D1WKyrYQZ6T9apBfAlVC4nHcBkqK70GgC5UBO9rrQBuc+bgTX0+X3THeg6dOjoKXQBXYcOHTr6B91xSfiLj+uVDLrmiCDttYwMcJL7HREWIUQcAHi93iWSJDURQuxQRXI7gEqoxECz9uiRWC0ujg0MFMiDYwObNJLHHxvoL4qbe9uxcLkghLhkWd4nhMhmjI3V3BX3M8aaKKWHJEnK7U58dkewWCyMSxJfX1sbE8cY/Q+DwXteCHKAMfKdFPwAACAASURBVOlJgMdLEi/zeMw/HTmy/LdVVfEf/+MfE+bMmfNFYHSfEMIQfK8DyJ6IgOUC578XBkXZOZubmw11JSW3JUoSl5qbLxANcUYjIjinVqvVBACUUjz9s59lv/Tss+H3NTREjwVQDKA2IsL56I9/vK+ysjIszOMxjo+PtwEAdTis1G630rNnh3ObLeo/x4wxp40YQQ7U1sp7P/vsho8HDXIuW7asEgDsdrspFEBkENGXJElcKIpss9kMfSGgU0ox+cYbT6799NMZdwKiDhCVnJNyziUuSWyY2ezZaLNFIDGxNjQ01MA5j+GcX5irJ4SI5ZzHlJeXh9TU1MBkMmH48OGuyMjIjlISrISQbpEIkiSdlyRpgxBim58UYow9pL33/KRQj2ZfdobOXOlCiPnddKVfawK6gPoZdVp7hAOYBJUYmgjVVpML1al+OaTwNRXhrjmodAe6Dh06rnZ0VyAWopdnYl8l8NeL19S1a83AFnHxPPIQqIlAoYyxia2trSmU0iZKqUsI4Ya6F3CizUXe0/StzpzN/hFV5oDD/c2XVkJIMQLSt6CmFV3ZTegFaElexVq8eyxjbJoQYqaiKDcQQgoppQclSTp/uef3u89drpEmzstJa2uLidLlrKXlt5YVK9bMEuI2Ul5+OnHt2rUdxoRrInnEJUTySzVyB44E8wHAgQMHonfuPHOT2fyftaGhYzxVVTviPv7449Sf/OQnZ/wneffd98bYbCHhknSAt7bmmtTnoRBFCZPef3/F2KysrK9Xr149uKrKFJuQcG9dU9Mud0PDOvd77/1XdXZ29ozKSh41dKjR8vLLv9kTERGRG1xDvPnm2xle7yhDZOQtdoAiP/+/+tyFHug+9/maDB5PiYXSBxhj1QZJGt/S0pIa+vLLL2e8++67+wIa59ulcAGIstvttLa2FiNGjOhMJLdq7u8u98qEEI8sy4eEEIc454M55xmc83mc83mEkJNaA3J5b/2dBLnSh2sNyJO76Uq/WiPcO1vHv0YTgN1QxfThUBuQFwFYCLUBORfqLPXLeU7XTIR7QBPMt4Lr0qFDx7cfuoCuQ4cOHf2D7mwA9Qj3qxBCCBmqIyES7UVys+aIoNpxfkeEG+oM5TJKaRLnfBTnPAVAuSRJ+ZTSs90pUoUaG9hurl4noq0/NtCKXo4N7G8QQrgsy4UAChljgzjn0zjnCzjncwkhx7SIbWuXJwpAVlZW00fJyY15VVUJ0QAMhKCAMckA0JmUKlWECIUxSZJlaUFcnHKitjatqalJRERERAbc77CAUwY7UALvd6fz3wEgJibGZxwwwJ5fXj5gamTkBQH9TEuLqclgUIYOHXqBFJw9e3bjkFWr1q1atSqtsrIyMiEszJM+cKCjubnZVF9fb2mhVNR5vXKiyaQAQJ3DYTnX1BQyxGBw3U/IOZMkRU0fOTLOUF4+MHf//geWLl36hcFgyM/IyKjd/eGHyt7WVuMcs9kLAFwIbPZ4TBFDhlQNHDiwz2agP/vss7k/KiuLeb+gYHgrIahSFOMUSeKZ0dH8/dbWAfvCw+lDP/pRis/n+0XAr7kAWFtbW5s+WrHC3nDwoDnK4VDsPl/r6ogI28RFi4ruueeeytDQ0CsqkAkhrbIs5wghcjjnKRopNDeIFCrrRVLoclzpV+sM9I7WQAfrOAB8pT2GQCWFZgOYB+AMVFLoLLpPuPQ5QROAPndtMMb6XKTXoUOHjn6AP8K9K3DoDvSrCppIHh4gkodp/98ftd5Z+ladLMvnGGMjGGOpjLFQQkiOLMs53Wli1ETbyOB6KaC52BBwuB3qPr6WEHIySLS9rGbdbwqU0kZK6WYhxG7GWDrnfCpjbCJjrFKLdz/Z0zrQ7z6PinrEcebM86lCfIdI0mymKDOk6urdiaNGPVDscu20fP75l+l33XWXwWAwRPuToALut3/uvIDaKG4lhJQTQnKD7neXe/c1az5Nc7lGmOPiMusIoTCb53t37lw9ctmy2jK/C33cuLENdrvtqJpy3QarlYSmpAy0ORwOafPm/aMlaRE3GuNYbOwt9vPnd8a9//77Z3btOlxhs81WzpwpCM3Ozp4/f/78uYSQfEmSciiltX73ucn0p1ZCJJjNi1pttrcj+tqFvmLFijS7nYQryja5tva9CMZGyISUcMBLW1vzo0JCvoOTJzfccO7cuRsTExM7cpKfIYQ0rlr1ccSxY+ctv/zlw18OHjzYbTQaefB8956CEAJJkiokSaoQQmxljE3mnGcwxh5hjDVoDch55ApGvgWtxwGcJYScFRfPSrd34kq/VgV0PzjUmugs1KSMSVDrpglQRXZ/A3JPGo76q87otwh3IXoQ46hDh47rGrqArkOHDh39g2470Ltx3LWIb3UMoxDCpJE9UQGOCL+TPHAeOYNK9rihRmY5ANh8Pp9t7dq1ETk5OYkmk0m54447SqdPn94ky3KuEGIL53w0Y2wmY+wBxlgjpXQ/pTSPEGIIiA1sF7mOS8cG+okHKyHE8W1wRPQ2JEmqkiRpHed8B2MsUwiRqSjKVELIWUrpQUppcXeu22g0ioeeeir3xbKyAeXNzVH7fD6TCyAKIRAmEz3n89HYgQOhDB48SamtRaTBAFmWY4QQjYQQq6IopZs2bZK+2L071O3xeNLT08see+yx4ssRbGVZFrMWLz7zxdtvx8pVVTFZkZEtNa2tho0OR6hp4sSyOXPmtIuOT01NdT/33HMF2dnZMWv+9rcpzbm5qQYATkCUtLYa36yqinty0KCGeKNR2d/YGHGQMWnxgAENFoDBbm+kNltjphCx2UVFUU6nc3FYWNj8m2++Oe+TVauqfpGTM2SZz2cYLElsl9dr/NJo9D7+8MOHrpRUCYT2dxXDOY8BEBsWFhbz3sqVMQUFBeHZ2dnyF1u3YlddnbSPMbN3wADfDUuX1tx7771nNQd5Oyf5P//5z6FNmzZNfDQ+vn7wwIHe5cePJ+cVFk7K/uqr9DeWL7dFDBzYmJmZWfa9733v9JgxYy7bMa6RQuckSTqnudInaaTQw9rfrt+V3luk0KVc6U7NlX4Kba70q8qBXltba/R6vTQhIaHVaDQGfwdeap1y7bEF6pz0KQDuh+pCOwaVGOqqmaa/Heh9PQOd6g50HTp0XAPoSYT7tbfB7Rrf6noxKH3LH7UehrZ55LTtUNEKtWayQU3f8ketN585c8awZcuW5Keffvo0pRTaYz/nPJoxNl0IMdPn880ihORJkrSfENIUlL4VPKIqUJQJFm0DR1Rdc9+jmjN4vxDiAOd8FGNsGuf8Hs65nVKaI0nSke7uWzds2DK0uTkkzOv9q8FuL46UpGcF0CIDiiTETAjhS5s8+W5aU3Ny8OHDhydPmzaNA2jW7m8JIaSxqanJ/sorryQ+9dRTh1JSUi472vvAgQPRubnWIWbzPK/bXWwEAItleGttbVhMoAv90UcfLX300UdLA3+3tLTU8t///Zcb6+tbQj/++GPNfX57EwDIcgQ3meZ71617I6OlJcISH/9ordW6JuJvf1tlnzVrVqPZbE5XFCUDQMWqVf/yut0xlpAQ1uJ2bzKp93ucyM9f16sudBE0gm3JkiVJkZGRDID59df/YWppqQHwngQIUCpTSbIwt9vn27Bhw+kf/vCHfvd+Oyd5Xl5exFdfnZ/Z1BRp3rRpS/LevfuHx8VFOZ977heHNm/enPzUU08VXenzJoS4tffefs55qhbvvoBzfhMh5LjWgFzZj670kwDqAn7lqhLQ7Xa7HBEREfwZ1VWN0QLga+0xGGqS1w0A5gIogjoW6wy6Fsf724HelwK6PvZKhw4dPYIuoOvQoUNH/6AnAvq3VkjuQ3yjJJjWfBraQWygBapIbgg41oc2kbwemkgOtZv3ImHOZrPJP3j00YUNhYXDUxmjDoD8/LPPpi184IGvnnvuuQKNVGqhlB7hnDcAGMY5v41zflvQqVqgkj1NQbGBVgDua1Ek7w4opQ5K6RdCiH2MsfGc8+laxHaDFlOY54/5017nkOCmhClTpsS898knMU8+9pj0nydP4jZKUeXx4KXWVjrZZOIzBg1qslVXt2woK7MYZs0qjIyM3AcAXq+XPPnEE/OqDh0aO1lRpEhCsHHfvtF7duwofXflyi2UUnHq1KlwIQTS0tJaYmJiuprHhgceeKBCURS6fdOmUdsaG8OZwaAMmjfv+LPPPFMoy/JFhaTL5aLr3nlnYlZtbeRDgwfXh0iSqG9tlf/X7TZtZ4yebWiINCsKrWKM1pnN7tuio9s6zQnBeYfDa2PsnNFo3E4ISQeQ8be33w555513GletXg1nc7OIGTasYsmiRScyMzMbOOfoiYgeLJIHOH9i0cFMckmSrOnp6cXp6emNTz/9tPXo0aNKTU0NnTlzZkNsbGynpObZ/fuTbzabvWnh4a2/ys1NPVJenvy4EGKz12tsdLsTI2tqEosLC0c+8+mnU2fdf3/2888/X9Dti+gEGil0QCMkh2ik0PwAUugIpbSij13pE4QQ44QQNZxzC7qe+XfFTwO9QGjU1NSYvtiyZaCntDSa+nxUxMW1jL3hhpqpU6c2oWfEiQfAYe2RCJUUygQwC6rAfhTASXR8X66pCHfOOYFOBunQoePqR7cj3LtxzLWIb1xAD0rfChTJL5W+1YD288ibcYnvrBdf/N9Zubklo1JTU+1Lliyp1s5nAGCglJZyzluEECOFEJMURZkS9OuBTtuzHcRRX5dpLUSNdz8lSdIpznmCJqTP4ZzP1lzVBymlF8RFIYQU6NoXQsQsXrwgaeLEyqi33vogrLFxCEJDq+D1fglFOQdCHhZlZVWIjx9RbbWOl994Y8X5zMzMzy0WS7vX+ZVXXslYt+7QeEVRWpYvX57j/znnHGfPng0dMWJES3dqjSNHjsRJEhdCrBFO5xqL/+ehoXAXFJxJgCoKdohPP/1scF1dXKTTaTMXFW2PZyxeqql5Lsb/74wptKHBF2M2D/EYDAksJuYee2XlzqiVK1fmPfXUU3u05vcsh8OVlpTk5V7vr4XX6/VwzrnJBC+giOPHj8f0REAPFsk7cO5fGMGWmppqffTRR60ASqdNm+auqqpy+3w+u8/nc/pdtQaDQdx88801kiR1uM/9/PNtg53OoeYBA6bZN2/+4/jy8voko7He9+MfPx1ZVORJGTBggOvee++97Lj/QGgNyGWSJJUJIUI1V3omY2wSY6yOUnpYkqT83kp56MqVzjlv7o11unoa2n97pc6ora01rlr16fBZs9L99VLgOt1do0J7bIXagJwOYBlUnikPagNyQye/e81EuCuKQq7X7wEdOnRcHnQBXYcOHTr6B343RacI2MRdl2SQ6ONZftr5I0T7eeRhUAVyC9ri5KA5IjxQ3Yw1aE/29CjC+qWXXprkLigY9WZEhC8jPJz7KJX/0dwctWLTpjtO3nXX7WlpaRfFBkItbCIBDAQAQkiBJEnZlNLOCprrHoQQRZblY5zzY0KIUYyxGZzzWznni6C+bhzqPTUF/JoTaiNCfVxc3Om/vPGG49///d8nvF5QMKjV55NPy7JSZLE48svL3ScIoe7Bgyuee/LJo/5f/uCDD4ZVHTo09hWTqfXGqKhWACjweg1Pnjgx7Pnnn88cGRnpMdbVhUIIkhMT4xoxd27lwoULA7veLwKlFA8//HDZsmXLKioqKixRUVG++Pj4zmZeY//+/bHy+fPRSwcObArRCJIBJpNyf2Jik40xOvWeewo45+Tm+Hj32y+9NPuDxsaoH8XHN0VIEj/uchk3eDymEfPmHQ8LC2sCsFsIsYdSOvbpp5+e+uMf/zg5Ly/PuW7FivDi7dsnv7Zly3hDSkrj/Y8/fmz27NmN/udwCZE8OCnBP77APzuyMdhJHoysrKxL3S4AahHM3G5jlNGoVLpchq+rqhL/Q5JYBWNE4lx+gVI2mVJRwTnZ09ISsvrDD2/44oYbzs+dO7exy5N3AxopVC5JUrkQYksAKTSRMVYfQAr1Vvw9FULYhBClQgiPEGK4ECIDakOCgxAyMciV3tu4IvLE7XbTLR9/nJpUUhKbGR/vsISF8TO1tWF5n302zGw2n504ceLlEic1UB3pOwCMgUoKLQWwGEABVDG9OuD4/vyu7a8Id11A16FDx9WOLmumgOOu16ZjoI+/w4LSt8KhRq0HjqjyH8ehNkG6AdRCrWVsUPfedvTwu08IYT506FBqYWHVJLc7yfz559vuXrRoUQulNHhkkhdqzXQGKq+ZAHWEVR2lNFuSpEJdIOkclNJaSukGzvmXjLGZQoiJWiOCHepraYZaN/mhALBOnz7d6nQ6K1977e+jw8KsEmO/Ei0tjlBgCiFkE/d6IU6ePCLJsk85e9YbvXHjxqT77ruv0n+SyspK87ZtBye53YMtO3cemVhWVlaYmprqBoDc3NzITz/NHrp4sbV8xowZgQJhh3jiiSeK5s4tqu7o3+Lj4zsVYktLSy1ffVU+NCrqwZaWluPmmBir8847p+wNbHjdv39/8t69JCw+/kkrABgMCYzS+fzzzzeNe+ihh0ojIyNPSJJ0YsWKd+JbW1uzCCEToYrcp7/44ovSd9/9KGHmzJm1wWv3RCSH2gTSGNAE0kg6cJJPmjQJkyZN6up2tUNeXl7EsWO25NjYu+2RkWPchw/XjPZ6Jxm93vOGvLyq0UKMI++/vyrz3nvv3dCjE3cDhJAWWZazhRBfc86HaQ3It3DOFxBCCrW6qVeEeyEEhBBMCHFeCNEqhBgGIBXaZwkhJEMIcRTtXem9jV7Z/x88eCi2qCgyTpZP8MmTJzcHJXf19LOuFcAR7REPNckrHcBMAOeg1kzHcXHsfX8I6P0R4U717wcdOnT0BLqArkOHDh39g+5sAL9xR8E3iF5xkQghJKhET2BsYCjU2MALjgj1UOGBKoY3QS0UbGhzkvcovk/r0A+MDfS7m6Ori4tj7oqMJOlJSeAAZMaUR+LilM2Vlfjss8+Kf/7znx/TXOUXxQYKISyMsQzO+TRFUSZr8eRf9+as5asRQp1lGNbB/faLtsaAwzlUtzOBSqoVEkLOUEqbCCEXBEZFUchPf/KTxa5Tp+IfkWVfnMHQ+pmiyF8oiuxOTy/OzMw8f88995QHitnZ+/YNyWBMutFiuUDSTDAafdM5x6E9e8Y9MGvWielDhjRLlIpjNTVh2Zs3D80dMMCTnp5u7+oazWYzHzly5IVEg8OHD0f+5S9/yaiqqoqNjIxs+e53v3vs3nvvPe92uyXKmBRpMLQTzqIMBsWgKKbp06c3pKWluQCgtbX1q5WvvnpDTm1tdCRA6mRZCZk4seQ/f/rTC25sQgiTZbkAQEFeXl7ax6+9dktmS0vKnJEjoXg8jq3nziWsf/vt4ampqdakpKQI9IJIfqWQZVlEDRvWmH/48BCHzydRRZFvNJk8j7vd4XMBsUCWGQPQwBh9NDS05ajTGfbhhx+OnTt37r7efi6EEJcsy19rUYVDNVJoUSApRCmt6urvVyPYooQQsdrjAskGldD0n8AHLYYUAKeUCkLIBCHEWCFEnVBnpXdILl7uJeIKCY2CgoJIU0VF1KLUVGuY0cgBID401NZSXBxz/PDh2AAB/XKhQBXMC6CSkekAJkN1ptdCJYUK0A+idgD6nHTSBXQdOnRcI/DPQNcd6B2jV+rFDtK3/CJ5V+lbDWgvkjsvZ11/GpRfOAzYw1u2bNkOIUYgOflJnpv7bOSRI0eapk6desQfs66lb7UE7qWEEOCcD2OMzeScL+Wc36QlUR3pLVfr1QohhCGwZgpqdI0IOJQBsGg/8wAopJTmU0prCCFOv2gbHR2Nfft2bgeA559/fvIHH3y2MCbmtHXBgoF5AAGl/m1nHAYNGtQuov31118fZ7NFhMfG/qXRbn8k4vXXXx+/fPnyHM459u07mlhSEhWXnZ3fmpWV1dxR8lYgjEajGDt2bKfvv+XLl4+prq4Of+WVVw4F/nz9+k+HNDUlhQ8dOqvWbB7otVoLw4YNG+YIHJW1bt3W8ZI0iDc3/zW8WfMqc95K6+sRvX79+uRHHnmkDAAsFkudxWLZJITYyRibqCjK1Pz8M6NdrjRl797smMmTJ5+Gum+PvRKRvLfhd58nJk60nT+/Ncrtli2yfLeiKG9Kra2jzRERD9jPnPnTkNWrVw/sLRd6MLREhGJJkoo55+Gc83RtLFY6Y6xaE9ILA+v1jiDaEuba1UsBdVMgL2CDWpueI4TIhJAIAAuFOiu9VGtA7q00r15zoNfW1hpzc+sTIiOnucvLj0UfO3YsKsCFfqU1Rh1UR/oOAKOhiulLANwCoBBq3VSF/qub+jwhTFEUvWbSoUNHj6AL6Dp06NDRP+i2gC7U2XHXG7pNgmmETnSQSB4Clewx+c+jOSL8ZE8tVILHT/bY0MNO3QDyIboT8sH//Bm0eHVCyNn66uqRUR6PJDU0OAljCoSABCC8tTXq3LlzVZIkdRovp0VEfyWE2M8Ym8A5n6HNWq6mlO6XJOn4tdo9q4nkEZcQyQP3MDao97yKEOKf8+aPu/cJIYzarOppnPN5AMYKIQ5oRTkDgI8//jilKj9/5N+NxtbZJpMPAJ7iHMscjnCXw2F+6qmnzgY/R6YokiX4hwBcQpBBXq9h9pAhzVQj96YOGuQ4f/Zs7PHc3JjuCOiB+Ne//pX86q9/vXSw0xl2E4ASAH85enRMTk7Olz/84Q9P7AoPd+1vbAybPWDAhXj2g1ZrOB0ypN7v6gCAJUuWVGdmZn72+eefJ9vtdtNNo0Y1LV68uMZPUAU5yWMO7N+fltzYGPrwuHEKlWUDIiMjUgcMwB9On8aBAwfCly5dWkop9Y8TaOxLkbwrLLj99rK1JSVxVZWVMR4ABV6v0cU5TSJESIBwCUFBCEJkWSQCOO5wdPTS9Ro0UqhEkqQSznlYACk0mTFWownphVA/ty6I5GhPsPk/UxRonykAjgcRbI5gMV4jwccQQgYLIeYJIVxCiOJecqVfsRDc1NRkjOWc+MVzPwaGhbWerqsL7Y01ApcDsBvAFwDSoJJCiwAshPq94P/u6Y/Z8X36Wa0L6Dp06LhGoEe4XwIBolqX1y7ap29FaPvqrtK3WnCF6VsBe/joDvbw0WgvaDmgioj1hJDTeXl53nXrvpjG+QuKyzXL1dSUFvXHP75qW7/+4y8vFeutpQH5913xWhLVTVo8+VFJkg5QSm09uY6rCcF7+KA6Ndi536jtKc8Fxds7CSFgjA3inE8XQozlnI8UQuRKknRI+x0AavNqQ0ODYdOmPRmU3ihcrqNh06ZNq77//vvPdfYc/e5zWf43xWAYwmR5mW/nzrcmlpWVFTY2NhrPnhWxgwfPsZaUbIvOycmJ6o4LvTNUV1cb33573U1eLzUuWbKn2J+cVVpaasnOrkiNiHjQRamM8PCRrY2N4yM2btw57MYbb2zwv8eWLl1UWFdXVxJ8XkIGYdq0aQ3aezwyoMk1BkDs0aNHDYWFNiQkzJfz83emVlRUpKakpHgA1AaNE2jURjP1ex3vd58bDOOUhoac0JMn/zZEiCmEsVDi9bpNwBLB2CTK+URDX7nQg6GNZNsrhNjHOU/TGpBv45wv0sYLHCaE2PwiOdoSz/wiuTngdA6o7/HzAbxAYydmBQLAPyt9omgbi3USap1wJeg1IfjgwUOxVmtcSFpaRkNFhTvqwIHC+AAXem/VMQyq6/w4gCiozcfpUGunOqjfF/2hIfXLDPRrlUPToUNH30AX0HXo0KGjf9CdDdp1HeGONuEbUImdYEeEXyS/QLoIIRRCiFsI4YYqlATOI3eghxBCmIPmY0dfgnzwC1r+wsw/a88R2DEenZzMtlZWTviu2awYNaHrtNcrnyRE3D1hQrcKM80VfEwIcYxzPlwjhe7inM//trsrHA6H9Oabb44qLS2NTklJsT355JOnoqOjFeCCyzZYJA8UEP3EnoBK4FkJIeWEkNwAAqI5uBgOBiHEK8tyjhDisHb/pnHO79RcwYclSTr89ddfp6RyTm8wGi90nYdSijsNBmX58ePDOjpvemZm5Y6cnDFnvF55pNGoAEC1otAcxuiiAQOsNEjYjDGZlHPNzaaOznUpvL58+fxZDkfYn8LCnNGUCkUI/L6lJXTVmjU3PP744yeHzplTtHrz5nHnKypikywW70mn05wbGuqZt2TJ2WD3xsCBA8UTTzzh4pybAaQJIaZ6vV4/+dDOSd5cXe0bazC0yi0tduL1euD1eiSDwTiEsUGNdXVhAEYIIZySJFVpbqBvDFlZWc145pnD2zdsSLXX1cW8ZrNFGmXZm8uY8XbOSbUQ1Gw2exqEEKcJEcOGDeuXcQgaMSMRQiq1z6phAFI0Uui2gEM52hpvTgeJ5PaeuFC0z6BDQogcQkiqEGIU1Fnp44UQNVfoSr9ikiYsLMx3jhB4FIWYA96f9S6X0TRokB0qCdbbpIkAcFZ7hAKYBDWmkAL4CdSZf8egCgZ9gT4X6TnngC6g69Ch4+qHHuF+abQT0EVb+pZ/HnmEaItat6Dj9K1mXHn6lj8px7+Hj0Z7kTxwD+9vdD1HCMkPbnQNPO+f//zXeU7nEBoTc6uLEIqQkMfcp08/M2Tjxo1J/lnoXYFSWkcp/YxzvosxNlUIkaUoyjRCyAlK6de9FQ/dV1i/fv1Ak8nEFi9e3K5O7KBO7WxkkgdtInmp/35re3VXVylIkiRVSZK0lnO+nTGWJYTI0O7fGUrpQUppCSEEr7766vjmZjkyIuIVR0vLf4W+++7Kqffdd9+5zhod/O7zqKgHbQAQHv5AS1PTx1H/93//Nz45eZSd8xEkNnaU22Yrt2Rn5yd2x4XeGV544YWpTueAMM4jyfLlf5o5e/bsjeq9/XRIfX1EVEwMHA0N2WHq9caxkycLk/bu3Rvnd6E/9thjpZ2I5DFCiLk+n+8iJznn3Lpu3VanYRNS3AAAIABJREFU1Tral5x8S0Nx8aGIdes+cz3zzI+TAAwRQghKaQWltPSbFO8qKytDLBav12jcqzQ0fBjucJyLMBh+7Gtt/adBiCGU0jsUj6fBHBZ2u/PMmfw+daF3AIPWyJEPoEGoY6qmKIqSGXRcC9T3eB0h5GRQ3dRtB7lWXxURQoq0z9AxhJDBhJBkceWu9F4Rgv3u8+jo+S5KJSQmTnaUl58OdKH3RY3RDOBLAHsADIMqoscDmAsgEWrdVNIH6wL9UDPpDnQdOnT0FLqArkOHDh39Az3CPQCaSB6ukS0RAOIARPp8vrsIIUYhhBxwrBeqi9wFoB5tZE8zAJd2rm6vK9To784cEYGuVBdUR4SVEFIKTdyilDYhKDbwUvi3H/zg2AsnTqQ+arVGLTaZvDbO6VpFkUPS0iq+//3vl3b7yeOCu8IfdZYQ5K44IknSwW+TuyI7OzvmqX/7tweczc3RiZRihxBk7Ucfed/64IPa8ePHh+HiGLtm7X6XdBBjd8VFjuYKLpIkqYhzHqeRajMVRblhyJAhNecplYLfTS4hCJWkDtd+/PHHz+7buXPUv505M2QhpcIAiJ2cU29iYmPUkCH1gQIh4xzn3G5DVEpKj6Iu9+zZE0vq6+PuM5m80ZQKAJAJwZMhIe4tLlfIBx98MPrFF1/MXR0f7/5q797BHqs1NGrkSOvS229vnjFjRqyiKCOCOvSD49b9JKbfSe6PW3fbPZ6MM7W144jJVOdvBuAeT8vpujqDyeU6TQiJEEKkK4oyA0CRJEk5lNKzPRF7exNZWVnNWVlZx25ZurTkhZ/9bFF9ScnAcqfT6GXMMF2WvZIs+7bYbBF1CQn1y59+urC31tU+V8L99zmoESQwLUFA/UypEUI4AIQDGAh1dnlNQFRhr0QHaq9DKSGkNMCVPkQIcZMQwhlACvWk+eaKv58mTJhgP/31144vysqis5KS7KFGIz/V0BBymlI+ZsqURqj3rC/fQy0AvoZKnMwHUAZgFoA5AIqgkkJn0LvESn/MQKfQySAdOnRc/ehWhLv2HXfN10xAu/StSCFEMgAoinITAI9oS98SQi2KPFBrmAa0NRZfbvqWLIKSt/xCOVSHov/++6OorYSQ4iCBvLm7e/ivvvoq5ujR0lGE3AyXa5NWkwm43fGW9977Z8btt9/++aVc6MGglDoppbuFEPsYY5O1JK8nGGPlkiTtp5Se+ab2rB1BCIHGxsbI//7v5x8wmYy48cYbj5pMpnbx9gGHu3CJPXxvPB/NFbxbCLFXS0Kbxhj7HmOs3ul0Hv788y8nEXK/kKQkbjI96SkufnzwJ598ktKRC726utq0bdvBSYoyWnY6P7pQizCWLG3btj996tTkU4MHZzYDQFJShr2k5NRlu9Crq6uNX35ZMBl4RMjyQFZQ8IcRe/bsiZ09e3ZjQ4PdkpjY3ASsvHB8SIjMTSbZ5PV6030+H0Vb/HdHcetWQkhRQI3qr1P52rVrBx04UDs7OvontS6XwWexLPZt2fKqNGdO3nvp6ekDOedTGWP3McYcWhLVUUppj+rC3sCtt95aM3fu3DoAePTRx+cbjRGtnP+eKIrNIMRd4PxrCeDC5zMbGRtKe9uFLtqPFAh0kseig7FgAAoJIUQIEQ8gAYCHEFIoSdIRSml9bz0v7XU8IIQ4SAgJdKWPF0JUX4YrvVe+n1T3eWzI8OHDGzlnMBrDGCEjxIEDJ/wu9L4UnAWAYu3xawCnoArpD0H9TvE3IPcm/9QfNRPRHeg6dOjoCXQBXYcOHTr6B93ZoPk3iteEm0JzXgbPIw+MDbzgiCCECM65pbW1dSyltFSSpHxCSA1UwqdHkcPi4ujvQEdEDABDwOEOqIRDLSHkFAA/8WDtLUf3nDlzGp5/+eXP3v3736e8dOrUYNlo9I2fOfP0L3/5y6OhoaGXLXZQSmsppZ8GuCsyFUWZTgg5rsW7d9oprigK2bx5c6LVajVNmzatYcyYMVdUvAt1Bnw7go0xFvO/v/3tkBSnU/5HSAjSJAnljOGRxkbz//f004lbdu3KlWXZGkw+XMnz6AkopQ2U0s1CiN2MsfSFCxdO3/rRR5aVimL8ntHokLxeT6mi0E8URRo/bdqpjs4RGRmpvLNy5abXXntt3M79+4dzxuiYzMzSXz/44OnsjRtTNxQXx06Oi2uRKRX59fWh5+PjnbdlZXXo1K6oqDC/8847o88cP55kDglpnbdgQfFDDz1U7nK5JEkIEkppu/dKCKVCBqjJZEoGEHrPPffE3H333TEIIB80R+olRfJL3aObFi0qX5WTM+yfNTXRt8bF2ZkQ+LS+PrIkMtL51Lx5pw0GQ7MQ4gvG2HjOeRZj7AHGWLMmBB/tLQKvp5gyZYp99aZNa9evXz9o8+bNI3YUFqYecDpDuCRxy5gxZf/z61/vSUpK6unnCtA2szNQJPc3JgR+rrRLS6ipqXG88cYbcXnZ2YmEEDFu2rTiZ555pjApKalVCEE55yO0qMI7tKjCPI0Uquute9KJK32cUGel1wohTgkhuuMuuWKSJiIiQplz992lezdtSimrrIygjFFvVJR7yNy55zIyMpp7Y41uwh+r/hnU2X/joDos7oNKTB+DSgz1RlqB7kDXoUOHju7B70C/biLcRfv0rQghRATa0rfMmkgOAODqh32roijDGWNWSukJrXmxGZeRoiK06O/gfTzUvU14wKE+qHubJgAnA/bvPU7K6QzFxcURBgNTKN3Mgc0Xfh4eDkdjozGcc06o1kzaExBCfAFJVKMZYzMYY/czxhoppQckScrrqnnxwIED0RMmTLBfSe0GdDgD3j+vOQZAzCeffGJyuThcrhasXr166oMPPlhHtHh79EGd2h0QQhRZlnOFELmc81TO+bQNGzbc4vGEIi7ucZfPB2o0fsfX2ppp7syFbrfb5djYUFtISJlH7Vtsux9ut8HocsUbOVeIw3HeCABeb7LclQt9586dA7Zu3Zr6xz/+MSdwPb/73Gx+yCVJ0cLpXBP26qt/mTNr1qzcF198TtZeg47SErgQ4pIieWf3SFEU8tlnO0cxlkXN5hSfEAoiIjLc584NG/DJJ+uGZWVlHQZQyDlPZIxlcc5naQ3wJyilOZTSiu425/cGQkJCOAB8//sP5qenF1Tl5OSkHDmCMZxvpv6/PUkyMKNR8lZVNQ9QFAWy3H35IKj5pt2oKrT/XGmFlpagGRYaCSHWI0eOKH/60yuT//jHP+wNHEfGOY9mjE3RXOnTAFRodedJ0kUiXXfRiSs9hRAySGtALhFCnEbXHNUVO9A9Hg89dep8jCSFsrKy96MC/81q5ZazZ8+GjRs3rj/rphIAqwEkQ413nwm1AbkY6qz007jyWqQ/HOh607EOHTp6BNIT554OHTp06Lg8OByOJVA7NY93dgxjLNrtdv/EZDL9w2AwlPXbk7sCaE7xaNEWG+gXyU1CCDMCuraFGhvod5Lb0X62HuOchzPGpgshMqFGHudKkvS15vgOXjc4NjBGex6Xig0MfDSRDmID+xqcc/TEOdETCHXO92TO+XSo96Bcu3/tHMF79uyJ/f2vf73AU1MzQGKM+kwm7/jZswte+fOfvzabzZ0SA524UPwEWySC5jUXFhY677vjjtR/AWyBJHGi2XSzGaO3CkGX/9///ePuu++u6pObcRkQQpAXXnjh9j3r1k0YoShyLOfiEOc+OTGx8q2VK9cMHTq0R2JwRUWFeffWrYNaioqiwTmMKSn2WQsXnu+oWeHEiRNh//GDH9wRUl09YAbUmIVcWWYTbr752B9eeqlo0YwZyxbX14e/GBuryJIkgVJpZXMz+avBgL+uX49hw4YFiuSN6EUXyooVK4Z8+cknk4xWazgIgTc21rb4+9/Pveeeey567bSZiVlCiPEAQAgpoJTmXGlUZktLi7RixYqhubm5SRaLxbdkyZKiefPmdVvUVBQFp0+fDouIiPClpKRcknAUQlguIZIHxu/boc1+DyDY/PP1LhTklZWV5n//3vfuCKmoSFooSZwJgW1CUDZ06Ln3PvpoY1xc3IXPIM55lJ8UgtoEcU4jhU70FikUdK3hQogxUGMlTaJ7rvSnoboPdlzp+oqikNLS0hCv10tTUlLcERER/mucBuAmAP97pWt0gRkAZgP4Q9DP46GSQpOgur3OQSWFTuDyZ8j/J1Qx/svL/P0usWTJkltycnL+JYT4c1+toUOHDh19DYfDMQDAS1Drg04FYZfLtQyAHBISsqq/ntuVQLSlb0VCHVHlF8nNAEK0mkpA5ei8aHOSO3Fx+hbhnI9hjM0CkAQ1xeYrbb8gOlg35BIjqkICDvegbURV4HiqC/Ox+xpatO5FoJSK3qyhGGPJnPMZ2j7IQwjJkWX5ECGkJfjY/Pz88O9978n7Z8/OLHzttdf2d3Vu0T6ZKDjxLAYXz4C3EkKsDoejacaMOdNttvuMgBORkZtsR48eeONKRfveRktLi3TDDbc8UVcXGxcami4BBIxxpbW1ghsMhZ6XX/7Fx0uXLu3W3r+6utr05pufjLbbjebgfwsP97Y+/vjSU4MHD/YE/xvnHIsXL11aVuYY+Jvf/GDDsmXLKoUQkfX19fHz5993h832oDk29meMUkh2+4eSwfBnrFnzGtLS0i44ydG2j7/iZu69e/fG/u53782220NCg/8tNtZh//vff78tOTn5wnUIIcxa3Z4F9T1RSyk9JElSQW/wE7m5uRHbtm1L+fnPf368O383iqIgPz8/gnN+0d9fQkKCp6P6SeNjOhPJIwMO9UEVxv01aqNWpzaik5ECDz308E0HDtSMv/POCV//6U9/yulgbYkxNppzngkgFYCbEHJMa0Bu7PKCewjNGDJMqK70WCEEF1270jMA3Abgt1eydnFxcYjb7ZaCf04IwYgRI5xGo/FXALYAOHwl63QDLwDYCLUm8sMItQE5HUAKVK4vTzvmctMB+rwO3LdvX+yDDz64zOfzxffVGjp06Li2oDvQdejQoaN/cNVGuGuuh2jN1R2J9o4Ic9thQkAlejwA6qCSXn6yx44u7oEWE7dDqDF7WZzzaYqiZAA4RwipAmDoRLBlUAvhpqDYwF6L/u4t9JV4DlyY831ICJGjuStmao7gRs2Rntfc3IwXf/7zxePq6+N+FhbmTJFlttXlMv95+/aM//mf/3H97ne/KwgWyAPueUTAcn4XihXA8aB77iCEiJMnTw7mijJ4mCQxwvmF134opZwoilRfX2/BtwiEEPGb3/xmw7qJE4/s3r17ssdgGPzkzJlxCxcuHBwWFjafc36wJ47gwYMHex75wQ+KrVargXOOQKE0EEII45tvvHHjoPr6QX8fNMgbazLJkCTDRrvd8PLBgze99dZb/4+9L4+Pqrrbf865M5N9T9j3PQhIEvYgICpacVfUVsW6vVatUrG2r7W/9rVSXlu17nWtyquoiKhoFVCoiCB7QPYlkLCEQMhkISGTZOac7++Pc25yM0xCQmZCEu7z+cwHhcm9d07uzJzv9/k+z3NRt8GD8f4PP9C+wkLnRU6n3O/zye85l53Hjds1cODARaFUek+fPv3AFVdccWT16tVJjDHKzMx0x8XFBSRzdWZiHhF9I4RI06r04UKIPE2kb28qEXzs2DHXnbfccuWJfft6pBKxPIA9vmjRyItvumnlk08+uakxx3A4HDjvvPNqBhdMtZWUMgnaat1Cklvvy3Ko+/woY8y8z02SvFHNrddeey3VdehQ5zlxcSe6OBwSAG7yeo3bcnK6v/nmmwMfe+yxGit5znmJtspcrt/DGVLKa6WUP7M0hYKW3R5AlT4Ap1elB00V4HA4qH///qc0qoN5jtOgPovAAgBLACwFMAiqKXQ1gJ8B2AbVFGrq8E+L2BHCVlPYsGGj7aNRFu5QtUdrq5kYFDker/8M5L5F+rlVUDVTOYAjqK2binGaYS2mIol2cM53SCl7CyHGSylvkFKeYCq2pZwxlkC1duvWAcByqLrJzRjbC0tE1dlyDrLiTDOvmwrDMA4bhjFfK1rHENFYr9ebqV2AVlv3Wy+//M/hxcUxcd99t3nonj17tgwYMOAknep45k+SW3ut5jB3HmNsq99Ad81+ctasWaNLS6tdhvGwD6hEaemCuKeeempYY/e7LQXGGA0f3m/XiRNl4ZxvN2JjY5NjY2M7ORzJ4VJO8Pbu3bsXER1rTA3euXPnqgcfvGWHtk6vA6fTKa31E9VmkicuWrSoX36+bwAwwLFs2Q+3XHvttQ7GmPHcc8+hqEjA4QCVlLwDIiIpK73l5fH88cf/uPPTTxd8dqYkeUMYMWJE8UMPnfgx0OuIioryderUqQ4BzRirdDgca4horZSyj5RylJTySinlFC0iWK8z688ITzzx1wt27Mjrk5qaWnT11Vfnn+75DocD6enppwwsaZI8TgjR3UKUmyS5NcbBB02OQ1muW4eLmzR88/333ydlZR0c4POlG8uWZQ3LycnZ5j/IzhgTDodjO4DtUsokIUQGEQ3XsWI5nPONhmHsClYfSA8m7WOM7dPDT4O1Kr0bEZVRYFV6UOqZvn37Vpzu8oJxnkacAwHOUw01ILwJKpIxHWoAeQxUrZQFVTs1ZQA55DWTz+fjralHaMOGjdYPm0C3YcOGjZZBoy3ciahFLdy1MiHK0uwxFRERUA0fp+W5XsaYh5Sa3I26JHmTbcAtRFYgRYQ5wd2DiHpAbbzzAWzXTZ6g2ga2F+im2k7DMHZqdcU4KeVUKeXkJUuW7DOKipKfSkrydA4Lc5HD4bgxIcFZ4HaH/Xvz5surqqqutJD81dA2agAOSSmLiKjY5XI1qhDOzMw8zp1O7wKv1/F7zmtI00+FcJBhiAkTJhxtyusqLS11hIWFyYZU8s0F5xw33HBDnqmuJqJITWKO9Pl86QByDMNY05Sc78TERC8RuaSUnayNB0ujLfrInj24PTERibGx4RDCCyF8Iw2j8kBOTvT7f/ubTDMMbzchjO+F4D86HJWJnToVjZk8eevs2bM3toQaKDEx0Tt16tRG/74YYxUOh2MVEf0opRygB2KusTSFNnDOSxpzrFmzZmUgO7vnh1FRFUOcTp8gwqsnT0a9PG9e5iWXXHJwwoQJAVUGVJuvl0R18/X8c+AroO7zQlZrj2k2fc5UbVyDnVlZPSYBZJLnANDb6RSZRNiycWM3qKZCHTDGpGEYOwzD2CGlTNRNoTSfzzcGQK5uCu0MclPIzEqPhmoK9SCiyQBOSilziGgnlCq9pZo0LWVF2NB5BJRrzHaooa00AMOhmkPHoZpCW6Duoeaeq9nQqiG7GWTDho22jlZt4a6V4v4keRTUYHGE33M90KpuBHDfauJ5uUkeWuslqnXfApQb2Pnq6VQMZW9sHXQtDsbepj2Bc17MOV9EKpJohB7gTmeM7WWMrd61a1f1unV70uLifk1e78dJn3/++fTf/OY3HKc6ntWJ7yGiIsMwipjKgD/t8GhhYaFz4cLFY4luBefdCQCEuAEff/x55n//939vaaoK3efzsVANI0RGRsq33357ufXvSLki9BVCjAFwidfrHadzvjecLue7PpKciBK9Xq/V4j4BgEFE+OKLxQSMppSUX1SvX/8Qvvvuu3WTJ0/et3Dh4su93or4sLA3TzgcYTVrFh4OVFcne0NBngNqTa644oom1bZATd2+zzCMfdqJagQpe/IxAPZpIn1PU3odX331Vaft24/09Xg6hL/11pyRV1555RcNDfFbhkHq1Et0ag68KVpwA9jlR5KXBasf8+abbw+vquoflpT0bFFJyfVxr7zyypBAKnQTnHM35/wbUrFsqToW6wYp5Uldd25sbN3ZGOjek5mVbqrSzaz0o0S0k4iOovXUM8E6B05znkIA3wBYBmAAVL10BYDLoGqpLChXr8acK9QEul0z2bBho0mwCXQbNmzYaBk0ZhNoFnRBbwZpRYSpII/TRVI0VLMnEqoBYLUN9AA4CeAo6ioiKqkJ0R8U2DbQWgRbbQM9qLVUO2A2ewCUEFFPKeV4AD0BOBhjhznnuTZxHhhEFK7XPJ4xdlT/HrpXVlYO7R4VhQ4dOsQKAIxIQgjfYJfL997Ro6K8vHxZQkLCcT3xXsEYQ15eXtiXCxd2z//pp06QMiY5NTX8Z1dddahfv34Nkka9evXyXHDxxev+tmhRZoHX6xzPuVwrJX8LwJgJEzYOHjy4UQMXS5cuTfno3XeHFe3f3wkOh+gzYsT+Xz/00FZrHlqooIngH4joR12Qj9Gq/mLO+VrDMDYznUFIRC6qq9w3J/QTAURbDlsJS9ablLKouKgoU548yR1C1KzJ348fT+js8/HXIiM96dHRJ31E+PvJk5Fvuly+l99+e8GwYcPKQv36mwvdFNptGMZurQ4YQUQjfD5fJmNsD+d8Hed8f0Pv4y2rVqXebhhiiNPpAwCDMfwqKurkgtLS2C+++KL/+PHjDb/BBJMkt+brWdfczIF3a4v7U2whgwnudIqKAJ/pJwE49Gtq8Oc5L9LOHNam0PVSygpLU+iUmIszhR6OqaNKNwxjMBGlEtExKaVpMRtKtHQGemNQCmW//j2APlBNoUv0YxdUU2g/6r/uppzrjGAr0G3YsNFO0FhHLolaYie4F6Dct+I1kWd134rQ/2YOPRNU/eJBrfvWCaia6bTuWwHO6yC/iCqqVZHHwxKNhVrCNocxtlGT40WayByjifRUItrAOT90OhLzXAVZYsGghod3QA1v9yGi/osWLYaUPdGt280oLHSKL754Le7mm2/e3qVLlw0W8rCO49m+ffsi33577sDp02/KGzhwYKOcl2bPnp1eXl4dzflAIcRXHAAYO49KSj5usgp97ty5PT755Mshr7/+4tIOHTq0yMCE3vNnG4aRLaVMFkKM1kPc4xlj2/UAcj4QkLA1I5NOySQHUKLXuMZlbsGCBWHff7/zwvDwmZVVVenesrJh8S+++HrEkSNHpMfjcrpcXavT0qJ2L1jw8eKWeO3BgnaiWkpEy4UQ50kpRwkhbhZClOpIpyzG2GmHNt98850Mr3ewMzr6/vKdO3/d68svv+x81VVX5ethn0AkeSICr7k1B968z0O6PzfV5xERsz2GEU9O502+ZcteC6hC9wdjzOdwOLYC2CqlTNF150ifzzceahhhgx5GCMp+vD5VOhFdTETlUkqgZeqZljiP+X3cmLUTAHbqRyzU8LE5hFwIpVb/Caocru9cIa2ZtAI9pOewYcNG+4JNoNuwYcNGyyDkFu6klOvxfiS5VRFhtQ2shFITlgI4jLrZek3K3dJFsJnz5q+I8M95My2RCxlje9B428BtRLRNStlPCJEphLhJW5Ov1Flh5xRpcJrBBH8b6gqodd4fHx/P84UYeNDjMXpERHB4vVX85MkTq4uKwliHDoXx8fGbDcOouVeLi4sd7zz//NCOe/d2vDY2tsLgnDatWNHvg5ycxF8++uimQJl0Vrz55pvf//a3v6388N//HvVWRUVURFTUyUsuu2z9s88+e9r8QEDlyb36xBMXZ5SUxNwdHV1xorLS9dW33w7/Y05O8ivvvvtNfVbiwYa2idtGRHt03lqGlPJSKeUUqOKPo66q2cyQdEMpe62E7Sn3eY/Bg5MW/vDD+ddERrI4zqmaCEsqKqLvBmhQWFgVADgYw8NRURUfl5fHzJ8/v9+wYcM2LViwoOu/P/00tfjw4aTolJTSi6ZO3X377bfnhDIqwIrq6mr2P//zP2nLvv46rbysLKZTt275t99115rp06cf8H+uVgcs0UTwUN0UulUIUaTt3TebZLaUEoWFha7o6Gift7ralehwMAoLC4dhOMgwHNwwHCler6tr166X+nw1t4DVMeGAdc1RT75eS2D0BRfsW7ZjR58NVVWuEWFh1QDwg8cTts4w5LUTJuQ29jjmPQhgm25MjiCiDJ/Plwlgv24K7Q5GM8DS2GREVKSHQ7pDZQy6GGMDABRRrSo92Ai5dZ/GmRD1BGCffkQCGAZFpt8K9R26GaoxVOr3cyF/TbYC3YYNG+0EpoV7Y553pjUToNy34qBqp1gA0RaS3Gken4h82n3Lg+C4b7moNje4obgk0xK5GMBuVjcu6URD3/eGYXwppfxOW5OP9Pl8Y5iKgvmxObbQbRVEZGiSPCnAmlttqE2FbRER5R48eDDss8++GxoV9RsHECbCw68py8ubYzzzzDOFL7744tr6zrdo0Tddt26N6LZo0bcVAwcO3N2Ya9yxY1cXwxAC+B3M62EMZBigbdu2dYHaW5wWlZWV/MMP/z38wAFHpzlz5vR79NFHdzTm54IJznkhY+xrKeV6KeUoIhrs8/mGQQ2aeKFqJn/CtojVjWIzCds697mUEm+/Pfdan2+sERaW7gWAqKi7KrKz7+3+z3++lSDlOB4Wdk3l5s2PDFq+fPmGSZMm1Yk+euKJJ4d5vT4+a9YTm0O/EqeiurqaXX75lddfdNGkXdYYJys0EfwTgJ+EEF20vfskKeUkxth2nZWe539cp9NJS5Ys6bVr17GBsbG/p+joCeFud0bEokXf3nbZZZc5GGOmqyBBfYa5GWO5egDHJMpLziaxaKrPExOnFANAdPTNJ4uL551Whe4Pzvlx7SyxVA8jjND9ozLGWJZhGFmc81Ms688ERBQmpYwmonIiOg41gNMPuh/DOb/QokoPNhqjDD+b5zkBYAWAH6DqyHQAk6EyzndDfa5l+x23JSzc7ZrJhg0bTYJNoNuwYcNGy+C0hYhlorde9ks3dOJJ2QbGUq1toKmIYKjNI6+EKlQLAZShttlT2pjr8Tuv1TawRkFumRRvKOfNtFtvlm0gYwzmZLu2Js+UUl4tpZysM743tidbQrO5Z5LkFqLcnBT3zzMsYowdZ7U21Oaa15Dcl1xyifHqP/5x429zczs9EB9P3WNiIpdVV3dcHB0trrn11j2GYdQhlJYIjPS/AAAgAElEQVQvX94hat++lDt79Toe63JJAEhLSjr58v79Hf7zn/90+uUvf5nb0GtwOBz0/PPPr6n++9/XFhQUhHXo0KHK5XI1uiD6+IMPUs8rKYmZ1aXLcYcmQEdXV3vu37+/y4IFC7rfeeedOY09VmMRQEluXXN/JbkXikQzABzViuCtnPMmqeMfnDEj69Fdu7rfWFiYOIFzcUQIXkTEklyu6giLSjlMnYw8Ho/jzTff7PPFq69OGlVZGTbc5aree/x4wsK9e7vn5+ev/sMf/hCwIRNsTL/ttks2r1gx4hoipHJOS3bv7vuXxx7rlZOTsyghPr6yKC8vNjo5ueLCiy7Ky8jIKAUAxpjX4XBkEVGWlLK7bgpdIqW8CEDu4sWLKxd+9FGn0iNHosNiYhzJPXs6luzbh5uio+FijJgQvs0nT4psourLevbcaBjGbs65G8DJs0WSN4T77rtvz8Y1a3ret2VL//PLyyMlgC2GIbqOGrXjjjvu2H8mx+ScF3LOF/s1hW4UQpRbVOn+BG4dWD5fTIt7Mw/etGs0P9Mlau0as5mCgzF2nlalF+imkH9WenPQViwPKwCs0Y+uUE2hsQAmQhHsm6DU6SII5zotpJQcdjPIhg0bbR9BsXAn5b4VQ3Wt1mNQm0ceEvctfe4Iy5BrAtUlbK0Dl1XQw8QADvuR5OXNUXtyzsu1mnWltiYfo63JdxiGscpUA7cXkFLv1zeYEIdTs5qLAOwMMJhQs+Z///szFxw54vIkJg73lJfnRYeHIz4+fiqtWzcvMzs7O7dfv36nrGF2dnbk+vXHuzmdab6srE1dd+7cmZeamnraQYtPP/1k4bFjxwIqpjt37tzoYcX333+/1+HDvANjY2nx4uWDb7+9IDtUKvR6lORW62+rxX25/v9YAJWMsS2c8/Wc82NNIWw/+uijHvv3l3cl4qyk5Lkat6nyckSdOFEeGxt7/wmHI8NXXv5qzEsv/XPEpEmTatZ027ZtMf/+99p0IobrrsvKSU9Pb3CvHAq88sorA7dv3z/gwIG8Lvfcc89u075+4cKFnX0+H7/++uvrEOOGYRwxDONzIvpGCJGm9/zDhBCFjLEDROSZM2dO702btic888xsY8mSpWGcD0OHDpNAxERs7N1i1aqH+erVqzdecMEF5lB3cWsUH6xZsyZBZZ8PNIqKHos3/97rjXEtW5Y17NixYz917NixSfeyrjs3A9gspeykY7HG+ny+CYyxvTpiIPt0n7X686WO45lZO6HuZ/pJqM+XvQDKGGNhnPNkU5VOKit9F5qWCd7gSzQvMUjHC9V5CECOfkRADSCnAfgF1PetOYBcgpaxcLcz0G3YsNEk2AS6DRs2bLQMGm3hrqdYO6NuHrmpiHChriKikogqoBo8VpK8yRO1FNg20Np4sNoGFuviy7QNNBsPJS2xGTUM47BhGPO0EjNTSnmxlHICY2ydw+FY2xh7s9YAIgLVqvfrqMlxqnq/DKogO8oY22GZFG/0YEJUVJR4+uWX//3EH/4w6eF9+7rz8nLhSkiQ1919t7zjjjtGeb3eQdqafCNjrCrv0KHovgBM8hwAwgyDBoWFVa/buzcBQG5jzutyuahbt25NtsouyM7uNDUiosphIUa7u1y+gUR8z549iVBFWJNhUf9YlSj1keRmk81sOhRpwtbDGAMRObSiejQR/czn843Qa7iFMdYoN4f09PTSl99999O33npr8NJt27q6oqKqkg4c6PhtQUHybURw6tf/bXW1q8AwxO2jRh354NVXM6+srnY+0qGDm+t/f6eoKPbthQvTD0+fnn0m690UrFy5MnHDypXpTzMm73K5fADwEBGuqapyffWvf025sV+/A+lOp++A12vMXb261+Ff/nLzlVdeWWFtPqCWsAUAx8qVK/t99PzzGHfyJGVGR3sPFRRUvVtaWr2qspLfum+fuMzhqC6Ukn9OxGNTU/ddfvnlywzDaNX2a1FRUeKd99//5r333tuzZs2a7pxzumXcuEO33HJLblOGSQLBqlCRUnbUTaHRPp/vArMpxBg7ZLnXrSR5Eup+vjRJiUIqAiSVMdaTiC6EykrPpeCo0luKQA+mwiFPP5YAGAxFpt8ARcj8pM8V0nvVVqDbsGGjnaBJFu76O84/jzySiMKtxyCiKqjP5BKoHFZrHnlT3bdARNEB9u+m3Xq45emmE5Q1Rqa4pRxyGGOVDodjJRGtEUIMl1KO8/l8/wVlabyKc57TGgcQA4GInAGGXAOp972o3b9bM+CLWBOymrOytvZzOl2+srIbnWVlqOKcVbtcrvCwMEfk1q1b7+nZs+cOzvmPhmHUDBEuXvxt1xMnukb07TutYN8+d8clS5Z1TU1NPa0KPTIyUp7Oovp0qKys5J9/vmwIMJk6dLizuKBgY2JzVegWkrxOPFU9JLk5NL/fso+ss5fUJOZoIhqm66efDMNYyzk/3pjrcTqdsmfP6Hwpf2Jqe6Xej/v353evrJwU4XSO9qnn3Vftr0J/661/nVde3i0SMPCvf717Xnp6+o9nui5ngurqajZnzofjiUaw8vIt0U899dTwZ555Zn1BQYHro4+WDxECfMyYMYVdu3at0v2gOvWS/m/zsyWZiJKLi4vx+efLRUGB1/fqq6/uX7x4Xfeqqin84MGPK9XcD8HjiY545pkXwiZNmrSnJV9vUxEXF+cdPbrXTp/vpKHcv2sREdGv2ul0NmvPzjk/yjn/ioi+1ffeCCHEL7RFfhbnfDMAh1kzWfoDSVC9MBNVsMSx6f+uNxqMiBirzUofSkSDLQPIzR1kaikC3ewDBuM8HgBr9aMLVM00GsAEqN6OB3bslQ0bNloZbALdhg0bNloGNZtArWKIIYvVOoAYKWUkAAghpgA4zjkXWhFRCdV8OY66VusVZ6CIsKpr/RURgWwDiwDsCjCd3ypIK63EXKhtCscS0Viv1ztOW3Ot5pyXnO1r9Gs8BCLJG1Lvm2te3Fgy9nQYNmxY2YJ///vLrKysuOPHj4dlZGQUJycne6WUHYQQY6WUF0kpJzLGsqJjY48UKwVPHRR7vQ5XbGworJvrICw62nPM7bYWrPARoQBAp+joBocGLE02f7vGJDSRJG8ImsTcRESbpJS9pJSjpZRXSCkv0vfhusZYxKWmppY/++yz68z/X7BgQdf/feyx66aVlcVc6nD48qTknwOsZ3r6jk6dOnmoqCh2akxMObdc31WxseXvFxfHr1y5MuXmm28+dLpzNgdLlizpGSYEv83lqvk9MAA9AERUVkbcN2BAWFJycgw5nRFfHzwYs2XjxozyCy/k4eHhgEXVzBjbwxhzE1HRa88+O3Lc8eNdf9e3r4OFh0cgMdEYVFBQ/quCAnGke/e8548eTXaFh1elX3DBzscff3xTeHj4Wfkc8vl87PPPP+9y7NixyOHDh7szMzMbtEMNDw+X99xzz/577rnnjBTnpwMpZxJwznOllCeJqC8R9RRC9Pd7qulUcZQxtl2vv1t/vjQpDkEr49ZTbVb6QMMwgqVKbysK9ECohlJRbAaQjNrMPyeAMfrftyME1vc2gW7Dho12AlOBDqBB962eRBRXVVV1I+f8pP4ZM4/8OJrvvsVIuW8lBNi/J0B9rps4AfX9eowxttNv/x7y/XJjoPerG4goSwgxWEqZKYSYLoQ4YhjGSs75ruYo3oMFIgqzrHkdZTPq7t+t0T2HAqj3m30tb7zx0qdHjhyJ8P97zrlj4sSJXYhojBDiHiHEQcMwfty/f//h9euPd0tM/Hk55wYSEy8oy8qa22gVenNhqs8TEqaVOhwJ0um83Ld48bzTqtAbIMnNdW8SSd4QNIm5kJSLUoaUcqTP58uAiiNayznf29B9OG3atMPTpk07bP27d955p/cTT7zaw+XqUe3xvKsJZoHKytjwl17658hJkyYt2rZtW8yqVXsHRUTM9DDmwNq1Tw/Iysra3pIq9FdeeWXgsWMlHRyO2V4hPnAsXPjRuMceeyz/P//5z4Dq6m79HA6na8OGDX1TUlKioBzOTJiq5uOMsd3muhMRffjhh1MKCiJ7MzYwbOnSFT0jI1FpGF97ga9r1jAsDNXl5TGn3MctiZUrVyYOGDCgvKH7MDU1tfydd95ZHsrr0Pd6uL5nNxJRDwA9pJQXSikvtDzV7IW5AWwza6YzcT1jdbPSY6AGkHsQ0UXUfFV6W1Gg14cj+mEOIKfpPwnAzwBkATgW5HOaBHqr6GfasGGjbcAm0G3YsGGjBTBnzpwulZWVGdOnTx/gcDgiULsJZRZFxAnDMLYKIfpLKXsyxrYZhvG93qg3CppQt9oG1pDlaNg20Gw8mEVxWVtRIwAA5/yEzldeIYQYqW0KR+o1XMk5Lwjl+S1NtvpIcmvjoQRq3Q8wxjb5NR5aJNMbUKpn6/9zzgv0MMJ/hBCjiGjExIkTRy/asaNwRXGxIzMu7ghjDFmFhVE7nU4xesSIRikFmoOMSZOyl737bue0srKIzOhoTxUR+7/Cwvgj0dEVd15++UE/JYq1yeZPktdMikPlY7s1SR409Q9TEQO5hmHkSinjzTX0+XzjGGM7OedrOOeHGnuu66+/Pi88PHz+22+8kfH8vn3dI6KiKsZPnrz9j3/84+Zdu3ZFE+dUpiyba3BCSk6MyYiIiJDdR6RVVykpKbGSc17qcLiSHQ5OjLGjAPdUV6OPYcDVsWMfGRZWybzeivFJSYUrd+3Cjh07skaNGpXDAmQa5uXlhZXm5U2+wOkscbrdleR0uigqKm54hw6xAyor+bhbb/Xccsst355tpdSqVasS//Tooz/zHD2aYghhvOdyeXsPH777n2+8sSwhISGU626q7AKpIqzDT5VQ9/puAFX6Od0BGIyxw5zzjZzzfcFqkuvfRS5jLJdqVek9AEwmopNSyhzdFGqKI0JbJtCtKATwLYD/APgDVGPsCgCXQZHoWVAqyKBAE+h2M8iGDRttHfTXv/41fdq0aVF9+vQB1XXfEkzlkVdwzrcTUX+fz9cJKrpohWEYW5sy5Eu12dh16iULSW5V3Vn375v9SPIW2783F4wx6XA4thHRNillXyHEeB0D4+acr9IOSiEdxiKi8AC1aiCL+/qGXFtEvZ+enl7aAMF6mIjWSykHCiHGCSFu3rlzdxnnfYuSk4fuJAISE4dUZGf3aLQKvTmwqs/Dwnr4ACAh4boTBQVfJ86ZM6ffb3/7251+A91JaIAkh1p302ku6PnYjLGTDodjBRGt0gMdY4QQPxdCFOmM702skQ5rhYWFEVFR8BB9DuDzmr8PD0dVebknHKhVn3foMLWQMYZjx+Ylt4QKnbT1d3V1dfL8+QunGMYlRlRUmiFlCqus/Dh+7ty5dxUUCHTqNMEnpVG1bNki39ixYzcmJiYWmAPd9Q3h5ObmRsydu5hXVd17LDEx01FUtCH+b397MHLChAnFnPP1hmFsZow1y9UgGMjOzo781a8emTZ0aL/9H34499tQn0/3wyLriamyihfMgZACItoDZS3eDUA8gDLO+Ra9hieDdW26x7bOOoAMpUofQkRHqemq9LZOoJvwQllK/ARgKoChAM4DMApAPlTNtBVBGkAWQoTcGcyGDRvtCzaBbsOGDRstgC+//LIgOzs74bnnnut8//33599xxx0bwsLCjkE1Y2qaFIZhgHPuFEKkSykzfT7fUE0C/2CSwFRr+x1IEeGfjW1OLBezWttAs/FwWnVtWwNjzKOL8dU6J2ycz+e7jyk745Wc84Nn+po1gVXH4h6Bm2wSQIluNFgt7t2aOGzVCkHOeRnnfBkRrejevXva+Vdfnbl5+fK0bfn5/apLS/OPMnas68UX773wwgtDTqD/6le/2vvY/v3Jf9uwYXBKUVFytWEYlV274s577/UMGTLkbq/X26IkeWPBOS/hnH9DRMuFEOdLKUcLIe4UQuRzztcYhrG9MffB1KlTj06dOvUr/79PS0srjezR4+j7e/b0TA0Lq441DPJIyd4uLo6jTp3cl1xySbMmtf0aD9Ymm9l4cN188814+9VX8URZmfGMYYgwInnM6/Vtk9JxQ1zcifj8/M1MH4hVVxsVpaXxlZWVBznnxYHOGR8f72MulzfP4wkDAOb1VrOSkuPFQhQVVVZ2SExMjNZKqULdFPqppVVdFRUV/I8zZ07te/Roxz9FRZ0c4HD4llZWhj25fv2Q3//+9xVvvPHGD805PtXNkkzyW/cE1M3vdGtVxBY/VcQp9zoRhVmsCm8RQpRwzjfq5mQwm0I1qnQAvQAMMAxjMNWq0ncT0eHTHAZoWQK9JZon5nt9HYA9UIp0U5leCJX59xPU9/UZw1ag27Bhoz1g+vTplW632/viiy8OuPTSS4tnzpy5a/DgwbugSL0a9y3OOTjny4QQXaWUE6SU10gpJ3LOV+o9ggBqHIkCZWMnoG42tkCtunavH0EeNOKwtUAPfu4zDGOfXsPxUsqrpJQXcs5X6zinM8rptewj6yPJrWrYQBb3RdoO+ayTfw2BMUaGYewyDGNXfn5+740bcy+LjU3o4fXO7VpeXnWspKTymJQnaetWd6fc3NyDvXr1Ctnr+fzzz7seOyYSibaEud33dTEM7jAM7oiIiHRu3557VVVV1XWcc/+BbncoSfLGgDEmHA7HViLaKqXsJqUcI6W8VEo5mTG2SavSA9YOJh599NEdDdnUW9XnnKv0osjImzzBUqHrQRxrNFiN9Tr0kOuSJUtw+HABwsLekURMEnUnIW5izz//SnXv3r843KPHqHzGHLRnz+KUTz755ND999+/93Tnff311wefOJEUk5R0fbEQMVRaOgIvvfSv0szMzAoAF+s13GoYxnrOeXOtws8YL7zwwvmlpRGxGzZkD1q5cuXG8ePHN+jc1ViQcqtIlFLWEOSW4WJrP+wE1L1+UIsXrDF40u+YkFJ2l1KO0Kr0yXoQfgPnPDdY/QQ9yJzDGMuhuqr0yURUQbWq9NPVui1NoLfEZ4PpvvkygP5QFu+XA7gUagB5E4ADzTmBz+ezayYbNmw0CTaBbsOGDRstgCVLlizasmVL59mzZz/0+uuvP/TCCy9cPn369B0PPvjg99HR0XWa5owxr2EY6xhj2VLKEUR0vs/nGwJlRegFEIPAtoH5TNnyms2fojNtfLR1MMa8DodjHRFtEEIM0TaFdwghDmlFekB7OD8lir9tYDxO32QrCqSubYsw13DcuHHre/XqNezAgQNjDcMYcFmXLsmdO3cuZYwdgiLymo36lOSc88Snnn46Jjs7G3v37oXT6fSmpaW5U1JSCgHUWNidDZK8MWCMVTscjvVEtEGrfMZIKa+VUk7RJPCGMyEwOef49aOPrv7744/H/OL48cSBUIFhx+PjT9z90EMrIyMjG3X/aQWQNV/PSpJb8zvLoBoPR5iOFkhOTnbfetddHV97/vnLvy0vd/QDsFFKxsLCKn0pKcellGTo38c3x47FV6eknEhLS6s3UiEqKkoMGjt29/zFi0ec5/G4hoeHV5dIyZ8tKoo9ER9fMGLEiHmGYXSRUo6UUl6mLfJ/0k2hkA9zAMD8+fN7VBcUJP9vdHR5f4dDAMAVERFVR6Q0nlu16ryioqI1iYmJDUYt6KZyVABVhH+kg9XmvuZe1382Or8TABhjVdq2dYNuTmZIKSdKKS9kjO2yNIVCoUqPgmoK9QIwqZGq9PaiQA90rhMAVgD4AWrIIB3AZAAXQZHrWQD24QwaVDaBbsOGjfaAzz77TLrd7swXXnhh1DfffDNrypQpE8ePH9/j4Ycf/mH06NHZ/s83DCOPc/6ZdvAaLaW8Ukp5GdTexYVTbb9NRfMRv/17k75b2xMMw8gzDGOelDJZCJEppbxYSjmBMbbe4XCsDbRXtexn/AeLzX2klcAyI2SOM+WQYxLkxSxAZnBbRFxc3IG0tG4LiSguNjb2vPDw8IGMsU6VlZX7PB5PTkpKSlBqcjo1GiwJQOKECROSHA5HIiyuCT6fr8Ln8xVHR0eXcs5zDMM4KyR5Y6AHOg4bhvGJlDJWCDGSiDJ8Pt9oxtge7eR1Ri5Uc+b838DS0qgYw1gpKivXhgMAkY/5fJHGe+/NHZSenr72dMewDNIHIsmtgzhe1Fp/1wy5/vWvT0+prHR0qa7+a80+TcpClJWFhZeUdA/r0ydKAkBU1OTK5cvn9bn66ryDXbt2rZc8zc3NjVi6dNMQl+ter2HEEADExt5WtmfP97Fz58798a677lqiBRkjfD5fGoDDWtm/oyWH+bOzsyOXLdsw3OW6v7q6+mvXK6+8ljF+/PhGq9BNBb9lza0kudWtwhSNFDAVo2GNqWp0DJ6+Dw8ZhnGIiBbrQfgRQojbtUvHxmAr+1lgVfp5RHReI1Tp7UWB7n8ugqqFdutHDIDzoeqm8wEUQRHpm6G+X5oEIQRv7aIWGzZstC6Y4iAbNmzYsNFC2LRpU9gzzzxz36pVqx4uKSnpMGnSpLxevXqJI0eOxOXn58fcd999vgsvvDAadW0DK6CaQE6ofPRNnPNs1sZsA88W9DRxfyHEeKiY5mLdwCkDkGAhr6wFsE8/z41aZYQ1B/6c+wIVQnSTUo4jokFQDgbrHQ7H+saQwH4keZ2MPaiiyIQZLWBdd3drJcmbCt2cHE1E5wPgTDlMrD0TZcDBgwfD582b1+fw4cOxKSkpJ6dNm7Z/4MCBdX4XROQioiQpZSCS3D9fz0rSWnMN6208rFy5MvHdd98dWlRUFNWvX7+CzMzMvO8++GBo9+PH4/o7nb4DXq+RExdXMfGXv9x09dVXN/gajx075nr4/vsvKt21q2eyEEYZgIr4+BP3PProf66//vo8yxrGCiEyiCgDqnmSaxjGeinlrrfeeqvP0qVL+wNgEyZM2HfvvffuDVZW+uzZs4cse/31S9fHxZ2wZs//WFXlvFMIx+uffPLu+eeffwKoY0/qbx2YBPVZbqIUWk3ut+4hbW4SUbjZFILK6i6yNIUqQnA+QOXVDgSQTMpJ5Vg9qvSLAAwB8EKwr8MPU6CUDa+E+DwA8GcAX0IR5P6IgLIqTAfQEep7aTNUY6hB1ZUVo0ePvvHQoUOPE9Fnzb9cGzZs2GgdePLJJzO+/vrrJzdt2nTxoEGDiidOnOguLy+PzMvLi4uJiXG++OKLHHUVzZVQtVMEFGG+m3O+yTCMAjQxu/Zchd5njSGiEVB1UTZj7DDUcKV1uNi6nynDqfXSOTvQrfdZI6SUowDEMOWGtroxJLAmyWP8hrnNuikBdS2ozWgB677d3RpJ8qaCiBxCiGFSytEAOgAo4Jyv1TEDje5/vPfeez3XrVvXJdC/paenH73jjjty9PnMSLY6UUl63eNR25sRqK1VrbVTvTF4Dz300NgDBw4kWf/O7XbH5ud3S+7Q4SJ3XFyXkwAgZRWvqspy3nxz14333XffKQNDJmbNmjV07ty1F0iZwhlz1PQkfL5Cx3nnOfYtXPjxv/Vr4lLKAUKIUQB6Q9XSGw3D2Mg5Lz18+HD4k08+OfIvf/nL2o4dOwb9ffrggw+O/eKL/eNiYxeVVVUtDSN6DO+88+xcqwrdbzjBP6YqznK4Grc5y9q79TBOyAZxdB+ppxa2DAZAjLHtegC50fFsTTxnDBGlQtVOYaSy0s0BZOtgRTiA3wP4GMDOoF9ILaIBPALgAwCndUdoJqYAGAClQA+EnlA102Co6AlzADkbjRxAfuutt/rOnj071ePxjGj+5dqwYeNcgE2g27Bhw0YLgTE2GsCdAPoC6AdF5DIAcLlc6Nq1K3Xv3t1zxx135P7sZz/L9Vc0ExEXQpwnpbwAQAqAHMMwvjcMo1kWRu0RAchaa7Mn1u/pZQDy/BoP57QS5XSQUiboxloaFAn8k2EYqxljpTZJ3jjoxlq6bqzFATioifRdTW141aPgDzSd74Ff40E3HYpYEO3Qd+7cGf3tN990KcrLi41OTq648KKL8jIyMhpljyilxBdffNFl+/btSfHx8ZU33HDDgfoaOkRkCCFSpZQjfT5fj5kzZniXff21kSqlNIhoC4A+gwfv/fzLLz+Niopq9pT5ggULuj77u99Nm+NyVY90ubxapuB4tqwsal5ysu/Tb77Jcblcphol0HCC/+OsD0DpplAPS1MIjLEduil0xpEXpzlnlG4K9QIQTkQVWpW+E4r4uBiqKfJi0E9eF5cB6APgnyE+DwPwJwALoYjxhtAZqik0FEq9lwNFpO/EaRw/Ro4ceVNeXt7vieiLZl+xDRs2bJxFMPXl81eoJno/qNqpRkmekpKCbt26eVNTU0v+8Y9/bEUARbOUMlEIcQERDQNQqW3J1wdzv9MeUA9Za7W5t7qeeQEUMMaO+tVMTVJ5nkvQe9UhUsqxUENyRzWRvh1AVDNJ8lYfDRYM6L1qLynlGD2I6dEk8HrO+YkmHsv/freS5AmozYKvcYLCqSR5UAbp58+f323Nmq2dA/1bevqgY7fc8ouD9f3sunXr4hcvXtwj0L/17t279LbbbjulP6SHuEcS0XAATsbY7kcffbRy3rwvzrv++suXv/zyy0HNg8/Ozo686qrbbq+qetAVHX1/BWNe48SJa2IvvTQh/8UXnytA3Zgq63CCGVNVM9StY6rO+gAUEUUJIYZLKTOgrvs453yDHuoIOolPRAxALyIaAHWfkh5A3kVER6AGxX4HYB6AXcE+vwWxAB4GMBeKqA4lLoX63j/dgHM4VL2UBlU/laN2ALnBmIDXXnut/9///vd+Ho9ndPMv14YNG+cCbALdhg0bNloIjLGrADwBZc+arR/7evbsmTt58uRLVq5c+ejRo0e733zzzbsefvjh5R06dAhod0xETEo5SAgxAUAnKOJtBed839kuKloSehrXP9PQVNb62zWa2dg1zR5dQKcT0VCoxtpabZ3fqvP2zjasZK2UsoMu6DqituFgIhBJXlRfTvO5Cq0MGCiEGA01URoczbsAACAASURBVF2q7d2zrPciKQs78373J8n9hxNqlMz6v4s45+5QTuefbTz33HOZrz7zzOS3nU5+hcsFJoT4T1WVuNHn4zfdfffXs2bN2nQmx9WqiAQiSvL5fIl3TZ8+0rd5c9z90dGsT3g4X1FejjlC4IYZM3y33377Mb/hBLPR1iaa9UQUaWkKJQIo1E2hn0LUFAICqNKllLEAugF4Kdjn9MPPoEj8V0N8Hg7g/wH4HCrrvDFwAkiFItN7Qg0WbIFqCh0N9AMZGRk/z8/Pn0lEXzX7im3YsGHjLIMx9gPUniYbltppxowZro0bNz6WlZV1xdChQ4tmzJixesqUKdtQj7WslDJeCJGphz69nPM1hmGsbc97In9YlLX+duvmkKt/NnYdBTmAE0TUS0o5BkAcYyybc76Sc37A3s/XDytZq12JekPtO6L8n4rAJLnpSNTuSfLGQg9xj9LvZ5ce+lxrVQLr/WS0VdGMwHFJBB3JhlNJ8nYRyRYIROQSQgw7fvz46MmTf5bsdkchIaGs8rvvFr/apUuXJg0k+B0XACLNmKr//d//HTZnzo89kpL+SQ5HlMEYWFnZN3A638X//d+LJf379z+OU93P2oTLn+7H9dYDyIMACMbYVs75Rs55XogGkKP1sHMPIgoHcFIIcRgqH/wjKKvzUCEOwG8AvA/1fRxKXAblltCU+qwTageQw6Ey0rOgBpBPGe565ZVXBjz77LO9PB7PuOZfrg0bNs4F2AS6DRs2bLQSuN1ux2OPPXbLihUr/jsvL6/PtGnT9sycOfO7zp07B5yg1NPYAzSR3hVAnibS97SXZgbV2iDXNHosxa+1+VCJ2kzDmsZDYxTNUso4IcRYUnbQxBjLMgxjNee8UYrZ9ghNkidYVRGW4YRTlORQNsMG1PRvLIBjnPNVhmFsb6/Nh1BACNFFSjleE4oA4Ia6t2NR18LOzPC0Tuef08MJl1x00bSu27cP+MLl8pJhOOBwOIgxNv3kScoePLjwq2++eYvVYyFKtXmS1mw9qyqiJtbB7XYXPTVrlmPTDz9EorqaWERExdhLL9305JNP/uh0OtvFplo3hXpZmkLSYlV4OFj3F9XNg+9CRP0ApJDKTfcxxpZYVOmhwOUAugN4PUTHN+EA8DiAz6BI8KYiEUpdMRxqOCwfqim0DZa1SU9P/8XRo0cfIqLFzb5iGzZs2Gjl+Mc//tH/yy+//Mv69euvGTRoUMlDDz20durUqVtQj4WrtiUfp/f7kjG2zuFwrGYhiC05G6BaG+RAJLlV4SkBlPgPFjeGrKVaN7TxUJbahw3DWKlrz3axB2oqLCR5Hectyx7yFCU5lCNUAlTdJHTtuYZzHnB43sapkFK6pJSjpJQjoeqkcqhYJAOnxgv4DyeYfxafy3Xq3XffPeGrr9ZPCAtbDJ/vUuO3v71TPPDAA5u0sr+gvp/TIoZEKWVNRJWlTxBuPm/atFt8+/a5ISXJWpBgzOebNm3yytmzZweKNWpzkFJGSynT9AByHJTLxAbDMLbWV3ueCfQwvbnu/YioOxHFAAhnjG0DsFqr0kOBeAAzALwHYH+IzmHiciinztfO4GcdqB1A7gXVq9oKVTfVRMm9+OKLg55//vmuHo/ngmZfrQ0bNs4J2AS6DRs2bLQyuN1u449//OONK1aseDw3N7f/9ddfnz1z5szl3bt3Px7o+ZpI7yOEmAi12TymifSdrb2Zob+DIjSBEogkt2YaVsDS6DH/W1s2Nls1TkSRPp9vFBGNhppo32oYxirOecB1b+uohyQ37aetJLlVwe8/nFDHSk3fi32FEGOh7DZPaGX/xraiwg01dKMt3tJoS7KsezxqyVqzocOhGp27GWM7taq5/FwkyRvCpPHjf5G2d2/vOWFhNY0K4tx4sLratTUtjc377LMqxtg2xlguACfVtWy0qlHqs2x0M0usw549e6IOHz4cMWzYsNLk5OR2a1uqm0KmKj0eajjGbAo16j1NRC6tvvJvtCVB2ZSbKIFa82rGGHHOnaRwnJRVoX9WenMxFWr47I0gH9cfTgB/APApVBPnTMGhMtvToGyNBYAdhw8f3t6lS5c96enptxYUFNxHRN82+4pt2LBho43g1Vdf7fXJJ5/8Zd26dTf069ev7IEHHlh37bXXbkI9RDoRRfl8vnFENBIAGGMbDMP4kXNe3qIXfgYgIsOPJLfuY6x7SIFaZa0/Sd5sZa3e7/cXQoyHqj0LOecrDcPY1h7V0g2Q5IEUzVYFv1VNfspwgpQyWqupR0CRYDs45z8ahhEqEqzNgYgi9DC9v5Lcfw/pg/o9eAEc5Jxv4Zwf0SR5u7snm4ucnJyISZMue6C6+iGX0/n/vNXVj7ni498XP/yw1BsXFxcNtYa7oO7nRL+9u1XEUIF6Yqr27t3rysnJ8XdbAACMHz/eHYx4rdYEPYDcT0qZQcqhz6v7SRs45wHdowIdg5RbSBLVHexORuA8+FIABudcQA2NnLTEYgWz95IA4CEAcwDkBvG4gTAVyomsuQPOCagdQI4BcLS6unpzQUHBrvnz53d56aWXOno8nknNPIcNGzbOEdgEug0bNmy0Urjdbv7nP//5mu+///5P+/btG3zNNdfsmzlz5vLevXsfq+9nhBA9tSK9D1Qm0w9nWwVsURnWZxsYbnl6OfxIcv9Mwxa4XpfOph4LIJYxtls3hYJN3oQcFpLcavt9OpLcqmg+hSRvLKSUHbSyfxiUmtRUV7R7Zb+forlGvW9Ro1hz3kyy1l8ZUQaACyEGa8vMLlDvh7WGYWwO5kR7e8CMGTPGLJs376LVDoevh2EAnPN8Ipbp9Tquve++it/97nfhqBszcBLAMUujx1z3knNZjVIfdFOor24KDYR6T5tNoXyyWN1bHslQnzXWSA1ro63Q2mhjfnnwRBRJKiu9N4AwIvLoptAuKPVWc3EFlPrrzSAcqyG4ADwG4BMA24N0zGiohlDaddddl5ifny979uxZuWLFituJ6JMgncOGDRs22gxef/31bvPnz39i/fr1N3fv3r3igQceWH/DDTds1MTCKSCiCJ/PN0YPzjqYylRe1dRM5WCD6kb2+A8Wx8HiioNT7afNR4vZIAshumv3pAFQg7OrdQxRm9qn+pHk1r376Uhy6/79jOzWicipI3TGQtUJB/VQxzmh7PdTNNchy1F3mN7sE/gPuBYzxrxSyhQhxGgiOh8AY4xt07Vno8jLcwn33HPPhC+/XHeBy7XJy3kSJzrKvd4RxowZtxY//PDDEQAiLU8XAAoZY8ctA8Zu3aM5Z6IwmgIpZayUMl1KmQ7Vc8njnG80DGMb1JBHpJQy2TqYQLUqfrNeFah1nKsZTgiUB6/7bb2pblZ6ARHtpOCo0hMBPAjgXSh79FAi2PUZhxJ2pH/77bcD/uu//ouPHTu24sCBAzk5OTnDiMiu+23YsHFa2AS6DRs2bLRyuN1u/uSTT05dvnz5n3fv3j3siiuuyHnkkUe+79+/f72bYSFENynlBCLqD0W6/WAYxpZQEUOkssasTQd/ktxqo1aGU9UQ5qPVNFuIyBBCDJVSZkJN/R7QjbW9rUn9qxttdZQolnWPtTw16CR5YyCljLGoK8KYsoJe3dbVFZZ7vj6S3KpoLvEjya35eqfdiGmlTzcp5RhS2WfVjLFNhmGs45wXh+xFtlLowRDrPZ/kdruTf3HDDV08Bw7wG4lgMIZ5RKBOnbwfffrpri5duhyDihpIkFKmQimPS7WaOou1EwvXUMK856WUPaSUw6Cs8VxQDR6O2qa+F7rRA7+Gz5k02nSt0kMT9ym6KRQMVfqVADoCeKsZx2gMwgH8HsB8ADuCffCFCxeO+uKLL8YsXbo0wev1SgBfAfgXgK+JqN26I9iwYcNGILzzzjudPvroo/9Zt27dLR07dqy+//77N/z85z9f1wCRHi6EGKWHFcMYY5u1LXnI9ld++xh/kty6d/eiNqLKv2Yqa8wesqWgyctMIhoKoIpzvs4wjHWtaX/VRJK8FHUHE5pFkjf2+qSUA4UQ46AiZop0zfQTY6xNf59b7vkk658IrGiuqZeg1sBc+0b1CYgoQg/Dj4J6Px3URPruc21I1nLP17ieud3ulIkTp/QtLR3IXK5ra55bXf0xEhMP+JYvX7IhMTGxAIAkot669jQYY7s45+s45wdaUy+ktYKInFLKZCnlEFKRWIlQny0SdYe6zc8a/0fJmXzGk8pKTwXQi4jCEBxVehKAXwN4B8DBMzxGY3EVVERI0Ouz/fv3p3zwwQcXf/311/1yc3M5gBwAbwN4t5k1pQ0bNto5bALdhg0bNtoI3G43mz179qXffffdn3fu3Jlx2WWXHXjkkUe+T01NrXezJ6XsLISYoDftJVpJvfkMp+MbajokQNnUmihFYJK8uK01AHQzY4AQ4gIo0q3F8739SHJ/y8aGSPIic0IcISTJGwOtrkjTDcoEqIGE1a1ZXUF1M5r9SfJE1N7z/o02K0keVEWzzvAcSSrDM4Iph4S1nPOc9tTMoLqKZn/rQOs9b1rYuY8ePXriqaeeSl793XedfD6fPC8jY9fjjz++esiQIWX+x9efjaOIaAiUSmU753x9MDO+2yqIKFzf8/6qiCTUveeLoTK4I6GsY716HdcYhlGvU0ozr82qSg8nogopZa5uCjVVlX411HDUv4J9nX6IAPA7AB8D2BmqkwwePPjOkpKSjwFMApAB4CiAYUTULmNIbNiwYaMhzJ8/P+Wdd975f2vXrr09MTFR3HvvvVm33377Gs65L9DzSTlQjZBSjgMQyRjbYhjGD1rt12SQUtXWpyRvyAXK+mhzkT1SyjjtQJUOAEw5UK1uKQeqAPWq1fb7dCS51Y3orNpLCyG6SinH6T1PJWNsvcPhWMcYO3k2r6shUK17gv/e3f+er4SlZoImzIOtaCYiLqUcJIQYDRU1UKoHO7Lak3Ja16uRAfbu5oCC9Z4v3rt374lbbrmjU0VFJXQuuSSSEgDCwpzVH3445720tLQTluOHaYeEkfp4BZzz9Vqc0WrED2cDul6NtwwoWB24rPe8B8q5gkP1QVxQWenrdCxWwO+lZl4boEj0gQCSiUjSmavSUwDcD0U2Hwrypfoj5PXZ7Nmzh77++utdvV7vTgA3Qg87E9EzoTqnDRs22jZsAt2GDRs22hjcbjd7+umnL1y6dOkT27dvH33xxRcfmjlz5ophw4bVa6ek7bQv0GRRmSaAs/zJbKrNXapPSW5Oywayr7NmvAW9CDjbIKUC7qltCvtBDST8qAcSmj0U0ESS3NpwaDUkeWOgBxIG6Zz07gDcFnXFWblvSOXr1UeSW/P1TqB2QtyqRmnxfD0icgghhkkpR0NNaRdoe/ctbeX9R3Wt7q3ZhqaK32pTajaX/S3sKppzz2uVSpqUcoQ+Z75uCoWkmdFa4Nfk9G/4WG0by6HW3Gq3fkpzWTfMM4goDcpe/LBW928PYVPIqkqHbgrtIaLGKiOugXqPvx3s6/NDJIBHAXwEYHeoTqIJ9JuJ6EfG2HAAlxHRU805JmMsGsCfAJwPlSOYDOB/iOgvzb9iGzZs2Ag9FixYkPjuu+8+vmbNmjtjYmJwzz33bL7zzjt/dDgcAffueuAzXTtQxTBlA/0D57wgwHPDzYgq6HrJsne3qmorcaqS3Ny7N2sf01pBRJFa2T8KStm/VTt5NXuoS+8foy1E7elIcv888lZBkjcGUsp4IcQYPZDA9WDH6mCs45mAiAwLYehPklszmq2DIe6zfc/rwdnRuhciGWM/GYaxlnNe2KIX0gxQXav7JL+1t0biWetVa81a3Jyhbt0L6SOlHKn339V6Hde3pXVsKnSNEU1EgSzXrdFsZr0ayIHLYzmeoZ0mRkANBHv0Om4404GtRrwGU5Xeg4jCAVRYYrEaM0zSAcB9UKR2qJXa10Ct6zuhOsGsWbPOf/vtt50ej+cKxlgsgJsBZBHRhuYc166bbNhov7AJdBs2bNhoo3C73ey5557L/Oabb57cunVr5sSJE/Nmzpz5Q0ZGxv76fkZKmaSJ9GFQys1cAGWMsXiqVZKbRYBpPR1IEdEmmg6hgpSyoxBiPBGdB8Cjict1p5tmP1dI8sZCRw2MI+WQ4NHqivWhUFcQUZhJkqO26WBO5lubDmbEgD9JXtQaCVXdzOgtpRytmxkepnI815/tHE/z+lCr4vdXRVibnBK1efD+Sv6Q25TqwY5+UspRekDGw5RN/oa2apNvGVCoafiY6w+lGDdRDZVtWNP00c0ed1OVJaTUPmZTqA+UaspsCoWkuUa1qvReACKo8ar0a6Gave+G4rosiAbwCIAPAewJ1UlSU1PvKi0tvYGI1gbrmIyxngA2AfgJ6trvBvCE3QiyYcNGW8PChQvj3n777cfWrFlzd1hYmPOee+7ZfPfdd69yuVwBv+dIRTmlSSkvgNqjH2GM5QNwWfYw1nxm03q6DlGuVbVNdUhpNyCl7E/Tyv5Yxtge7YjWoIqRTo1KarckeWNAKmogQw/OxjDG9nLOV4fCgYqUqjbOOuBqIWrjUTeyx7zf/eumVumeQERR2mliBNT+bJ8m0rNDXWs08voCDbiae3d/q3t/y2/z3g+5259lcDYDalA0R0eL7WkOSX82YekVBHLgMuMICbXRbNa1LzyTelVKmWgZQI4AkKsHkHeF4rOLalXpA6AGkBurSu8I4FdQueShjuG7Duo7991QneCJJ54YPmfOHObxeK4O5nHtusmGjfYLm0C3YcOGjTYOt9vNXnjhhVHffvvtk5s3b56UmZmZ/+tf//pHwzB8e/fu7Xbo0KGOU6ZM8Q0fPjwCtcSJtaIlAG4AOZzz45biq7StFkAtBSllgrYpTIOaZt9oGMZ6AA5rw8fSdAhEklubDkV68rjdkOSNgV7HMXoduSbcVjeVcCMil99wgpUkt6pqT6J+krzNWtHpdRyl19HFGNvBOV/TErbkFhVWIFWEVcXvn/NmbXK2is8bvY4j9TqG6ybl+tbSXLOC6rdtNBs+pmuIRO1QjlUVUYgQfd7oplC6XsdIAAc45xsNw9gRwqZQU1Tp10HZK84J9rX4IQbATAAfANgbqpMMHDjwnrKysqubq56oD4yxJADHYSspbNiw0YaxZMmSmNdee+3RNWvW/IoxFn7XXXdtueyyy7bv3r27S05OThcA0Q888EC1hagN9zuEB8AhzvlhqHxm04mo3VhChwJ6IGGIlHI8lCrvIOd8JWPsKLTzE9Uq+c39YyCS3H/f3uIuUGcTlnUcC0VqHdVOXtuaso+muq5z1gFj/4F6gcAkuTngGuyX2CLQ63ieHkjoAjU8uk47y4W0FqRa229rvWru2wOp+OuomoNtdd8c6HUcrJ0mugE4oQngrFAMxDcXpBwU6nPgsg4onERgB66QfN6QcpZLlVJmAOgJ5dKwyTCMjaEa5CaiKD2A3BNqAPlkA6r0TgDuBfAGgPxQXI8F10MNt4SsPvvTn/6U/v777/s8Hs91oTqHXTfZsNG+YBPoNmzYsNGGwRjrAmAEgH4A+oWHh6cLIYZ5vd4aRURYWBhmzZpVduuttx6BX86bVq+O09PDPsbYWofDseZcVko0Bn7T4YlSyg5Q6sc4v6faJHkToNUVI3QzI1oTlz9yznPNtaJaFb/VOtAkyaMth/PAQpJDr79W1la1+ItrQWi1z3C9jokAjuhc6mYRl35r79/w8R9QqE8V0epU/PWBlI3rEJ351xlAsW4KbWrpz0gicvqtfbJl7QPZNvo3fM7aQJRurplNoV5QTaHNuilUFKJz+qvSPVqVvgO1qvTroZpl/xeKa7AgFsDDAN4HsC9UJxkwYMC95eXllxPRplAc324E2bBhoy2DMcYBjIeumQzDGOh0OsdWVVV1IqKaDfnQoUPlkiVLDlnrJb2PLJZSDtCK9BQoxeX3hmHUG6Flo46SPJGIkqSUiVB51J0AOP2ebo0HO2dJ8sZA1/B9dCRWP6iItjXWfG+99rH1kOTWaDbTBcoalWT+Dk60tuHRYEKvY3ft5DUYypZ8k1ZTnzFxqQcUYshvoJtOtf0WqD+mqk31CrRN/kgiGgo1EL+dc76uJQa5rbCsfSDLdauQpGZAAadarp+1XoGUMkWr0s+HqvH2ayev3aGo5aiuKj2ZFAqIaDcRmXbtnQH8F4DXARwN9jX44QYoNf57oTrBH//4x4y5c+d6KisrbwzVOey6yYaN9gXH6Z9iw4YNGzZaMa4D8BKUjVd2ZWVlNoDve/To4UlPT79+48aNA3r06HG8V69eq51O5w6oCf46MAxjkZTyByHEOCIa6/V6xzJlpf1ja5wcbin4k+RksQ9EYCV5niZnYqGsiyOgbOFWGYaR1/KvoO2BMVbpcDhWSinX6py6EUKI24UQFVCkbBjqrn0ValW1ByyF7zltl8kYq3Y4HOuIaL2Usp8QYoyU8jop5RSd772xvve2VkVY73srSe6/9mazYR9qbb+L2suAAmPM63A4NhHRJillNynlSCnlZCnlhUxloq7jnOcDwNy5c3t8/MEHacXHjiUkde1a+Itbb908bdq0JmXEWRQpgQYUrGtfCWW5XsgY2+03pBBy28amgjEmHA7HNgDbpJTJuimU7vP5xkEREBs457uC2RRijFUwxjYS0UaoRv0AwzAGEtFAqlWlMwT4TgwBzCZpqGMIOFQj1oYNGzZsBMZiKELikBAiWwjxZVhYWO7EiRPHFBcXjz9y5EjYlClTdlRXV38fHR19yj6Jc76ViLZJKVOFEBOEEL8UQhw0DGMF53xfWyK7ggl/kpxqleQmUWslyksAFAH4iTFGRNQVSgFcqpXUm1rjXqa1gTEGwzD2c873CyF6SikzpZQXSSknQ6n1JdRgt7n2hNoBhRzG2AYLSX7Ous7pdTxkGMYhKWWsJoAzfD7fGMbYbu3klRvovU2BXaASLXt3q4NCsV7rvX4q/nYzoMA5z+ecf0FE3+pB7pFCiKFCiKNa3b/N+t6+//4Hx3Xt2qns8ccf33om5yOiiAYcuE6JCGOM7bKseyFrpTEDnPPjnPPFRLRMq/tHCCFuFEKUM8ayDMPI4pyXBut8eg1yGWO5RBQFIJUx1gvAJKpVpZfop7dU3RTS8wgh7JrJhg0bTYKtQLdhw4aNNgzGWAJUI+goBfhAf/rpp1O//PLLv2zcuPHKoUOHFs2YMWP1lClTtqGeTSkRRfp8vjFENBpqcnijYRirOOdloX0lZwcWktyqaA5EkntROx3ur4o4pfjSisthUspMfaxcwzBWnsvNNX/4kYX+SvI41E6H+6CKXxcUaZvNOc8yDOMY2thk/tmEnmYfrafZmSZe9wPgAVQR1rWvTxVRcS6uPansxDSdnRgHIO+NN97If+OZZwaN8PmMDMZoLRHf7HB4b3/wwUWPPPLITr+fBxFF+xPkZtMNtWSrD6darhfqIYU2PxxCyqpwsF7H7lDvZbMpVHK6nz/Dc0ZIKYcRUX8AMVLKOCjF/jtoOCu9uUgA8BCU0j0nVCfp27fvAx6PZxIRbQ/F8W0lhQ0bNto6GGN9AeQR0Sn2x5s2bQp75pln7v/xxx9/c+LEiZTbbrtt+4wZM76PjY0tD3QsrVwdIISYCEUA52kifU973B/5keQ1NusNkeSWWsmsmwIqyaWUHYUQmUQ0BICHc77WMIz17WG/EwxYiNpASnJrPjOgaiUn1H7yKOf8J875XtaO8uBDDVIOVEOllGOg3CYKGGM7oe7pBL+aNZALlL+Kv/hcHFAgIial7CulHKX33pVa3b9h+fLl/M47f3O7y2VUL1ny6Vu9e/cO+F6nwO5npuV6hOWpZQjswNVqIsKaA/0ZaarSXUy59G3Q7+2gkzr6874XEQ0joo6knL3iAXwK4IwGHpqAm6AGIOaG6gS///3vR82bN89dVVV1W6jOYddNNmy0L9gEug0bNmycA3jxxRf7ffbZZ39Zv379tYMGDSp56KGH1k6dOnULFDF5CkhZaY/WhaPz/7P35tFxVVe+//ecq8GDbGvyPGEbAwZswPMEdBI6kAANcXDCGJImPAIOxkNIIAkh3YROukk33bSTPDor7XT/enU/hjzIwAtkADxqsmzLg2zLkixP8qSSLcmSreGe/fvj7Ks6dXXL1lAlVUnns5YXiVSquro16O69v9/v5mJnS7wGG/HkEkPybERGrndpSN7JxxZKqWtc110K3Vw7KaXczFHa/f4PMIX367UPyRFW5mciclh41hgWmue/QQgBpdQo3jc/C3rdwHbHcfJjqcDuT/D13dAAV4TXdJDGzRsAnPA5Irzdhv3+ddodiEgqpaafO3duwec/85kp99bW4ntDhlxwWlsvKtdVa5uahv6/7OymP2zevGHIkCFZFBm5bjY6z0G7yf2R9/3GkXI5+L3tNYXSAZSzK/1gd5peFLnf0Itv9F735qqBFillnRDiHBGdIR1VGLQrvadkA3gawC8BxC3qd9q0aV+/cOHCzUS07/K37jq2EWSxWAYCRUVFaevWrfvq5s2bnw2FQmMefvjhfatWrfo4KyurPuj2PEif5rruLdCJJ6d4kL4v2f6Od3FIXofI1WCXHJJ3BqVUJiei3QSAhBDb+Fo/8Nz3N4hokDckR3gvvCewNAe13rDQPO/eoLaNB8A38p70LABHHMfZyuKOpHpN9hYUmTxn1k6jEHnuWwGcEUKc9g3KEzIFKlFQSmW5rjuX39uDV61aU/e73x0bolTIvffeGza/9tprpQHn3r8TvhlGzYTI9LO47q1PFEivabueBchjoXfOb+d/XTa8GP0CM+7eq1n9qwaa2cBwnl3p+9BxV3osuB9azP8/cbhvAMA3vvGNBW+//fbp5ubmR+P1GLZuslj6F3aAbrFYLAOIdevWXfHOO+/8TWFh4fJp06Y1rFixomjZsmXbEX2Qnu667jwuwAcJIXY5jrMpXntru4tvSB4RH4hLmm11yQAAIABJREFUD8nNxkPcYry4uTaFB+lToXcpb3Ecp0Qk0U7oIChyx5t/SJ6FyP16/n3w3n87PSxUSmW4rjufiOYBSBdClEoptzqOUx2XXzDB8TXbIvbsQQ8jPbxGp9d0OAsgUyl1HXTDt47j9dp3J1ouDRE5b7755nWvfOtbd/1pxAgxOT29fTVSaUsLPt/UhNf+679w4403Bu2Eb2909uGvkFBQeOf8XGjBUYPRFKr33dZz8kcMyI3PHXO/odls81z8ISFEKxENpvCu9CEU3pW+D3o1SizIAfB1aKd7PAb0AIApU6asbG5uXkhEZfG4f9sIslgsA4lQKJT6jW9848ubN2/+5qlTpybef//9+1evXv3xqFGjooqJXdedzI70KQDOSCk3OY6zN5EckAFDcv9e7EsNyc1BbdzczEQ0tK2tbQFf66dx/blFSlkTr8fsLYgojYiylVJBQ3JT5OddO/qH5J0e1LKQ+2rXdRdDp/3Uckx+yUAc9lI4/SzC0YyOg9oOu7EBtBLRNBZyp3D9WeA4TpfWNQ1UvEGt67qjdu3aNe+hh568Rsofiba2wxg+fD1++9s3kJ2dDehBbWACFwZo+lk0XNcdp5SaQ3rnfIrQKweKOfEwoq9CRKnG694vLvb6BQTgnHHuvfqpRgjhDecnE9HV0EN2kF6LtZ/Cu9JjwQN8LP8nhvcZwZo1axb96le/OtbS0vJYvB7D1k0WS//CDtAtFotlAPL6669PePvtt79fVFR0/4QJEy489dRTRcuXLy+WUgY2Q3iwMYcjyYcKvf93k5TyTG8dcxeH5F7h1WtD8s6ilBrLMYXXAWiUUuZzTGHC7o2myNhpc1jrnX//jregIXlM9+sFuCsOO46T1x/dFRQcX+c1fIKabf74wNpLDWr5NbmAdHSmEkKUOI5T0B8alT3FJxDxR65nFRQUiGcfewy/GToU0x3HBRFBSmdHS4t44MIF+vEvflG0dOnSj6wooWvwa3KO16gEcFIIcYb03nKv4eM5+dt3SyLSzd/p/YZcD00gomsAjOLPvFi50kcCeArAvwM42sP7isoVV1yxqqWlZQ4RVcTj/m0jyGKxDERCoVDKc88999CmTZueO378+NTly5eXrVmz5qOxY8dGFRO7rjtBKXUL6djiWh6k7+qtQXrAdfvlhuT+5C0v+rhPRX7stpzD1/rDhBD7OcnreF8e1+XgYZXpZm4XGAPIMG56AUa9BP1a8f5/TOtC13XHK6UWs2jwohCiKCUlpVAI0RjLx+lrogi7TYGl6aiNtqYq6oow0uaCG5VSC/j+jvMgvTSeopJkwScQaR/WwhjUPv/8t/HOO4cpM/P3LlGDqK+/zXn66btpxYoV+9hgMCBF8d2FX5OzlFJzAIwGcB7AcWgh8Ah0FIh4nzvmgNz73OnUa5h0pLsnQB5MRE0xdKU/BJ1O+EYP7ycqzzzzzOJ33323qqWl5X/F6zFs3WSx9C/sAN1isVgGMOvXrx/zxhtvvFhQUPDw6NGjW5588sltDz74YOElBukpvP93KYDhQoh9HFN4MhbHEzAkNxsOlxuSew2fPh+SdwalVDbHFN4IHUnuxRQG7lqMNxTerxfhYjaabf7Yab8jok/2jFE4Jn8RtLsixKKEpHJXsCvCXDVgDsmHGzdtRqSTPBSrZhvp/d5zlVLzAAwFUMGvyQ5K9v4GaSe/ed5zjc8er9HsDWprwM9BU1PT2duWLLnjtrNnh702ZEhjqhBoJsLXLlwYtmfKFPfXH3yQ7jiOy+6pIinlqb76HRMVbnZmGq/73CivfUA3PE8LvfuvOh5OOIp0pQ8moos9dKWPAvAkgF8AiJtTafLkyWtaW1tnEVFVPO7fNoIsFstAJhQKOd/97ne/sHHjxu9UVVVN//znP1++Zs2ajydOnBhVTMxisFtYnFXHw98dsfibFWVInh1w7QIED8m9uPWET8IhIocHREugrw2qHMfZzNenfXZMflG3ce6DrtvNQa03KO91cSXH5C8kotkAJF+f5vWmKL6nBNSsZvpZDjoKu71+QcxWJXH9Od113QXQ6XLnpZRFjuNsE0LEKsEoITFq1iBh9zDjpk3wCbvz8vLchx9+6m7gnyg9fXkLAFy48OPBI0b8XPzhD++05uTkDIcWJRRxekfCfz71NkbN2l4vkXaTm6YGjwYAh4UQFVLKkJSyRggRuG++m8cCAJNIu9JHkuYM9cyV/jB0CsSbsTpOP08//fTSX//612Wtra1PxesxbN1ksfQv7ADdYrFYLHjrrbdGrl+//oWCgoJHs7Oz3SeeeGL7o48+mi+lDCxajEbGzQCyhBBlUsqNnXEE8JDcG5Z0ZkhuupmTakjeGTiSfCHpmEJHCLGT99TFJSafiAZ7kd++Ibk/8rsewUPyuMY29gR2/Cwy3BWFKSkpRYniruBB4fBLuCK8F3Uborsi4h5fx+/v65RSC6H3q4U43n2nSOIdc9R5J7+3WzIocr2DQOT111+f9pO///u7J7S0pN9EhG1C4OSgQRfXfu977z788MMhpdQcdgUMA3CEm0L7EvV9FC/4sydaw8db9eA5gjq4yYkoRyk1l3RSghQ6qnCblPJQPAQeFN2VXkZEXdllPhrA1wD8HEDcXDWTJk36Rltb2wwiiqnLXQhxB7SgZji0COBNAG/xt98jIpuuYLFYBgyhUEi++OKL927YsOF7FRUV195zzz0Va9eu3TBlypSoYmKl1CjXdW/mv18N7LLcfjmhpzEkj0jeusyQ3D8oT4oheWcwRLNLoVe9nORzWRoPAW+AuNVfs3oX5N6KMHNQW5vIsdNENIjd/Qug3f3lUsqtfE3V14cHQDtrKdLNbNas5l7yehjihMtdt8capdRITvK6AYAQQuxmAXLSimYpvOohMIELYSe/V7P6I9drgwa1n//8Fz6zZUvVTY7zaVeIFH6sekG0Effdt3jDq6++Wuu67jwA06DfOzscx9kmpYy6OqM/Ynz2tMetGzXrUOOmXs0a4SYHcFEpNYvXYuVCp1tsY4NBXAQeFHalTwGQTpFrsboytH+Eb/92PI4TAJ566qmbf/e735W2trY+Hev7tnWTxdI/sQN0i8VisbTzq1/9KvuXv/zlt/Pz8x/LyMgQjz/++I7HHntsa0pKSmCDh4gk76y9GfrivIId6ccNZfKAH5J3Bm5kzOWh5RCh3f2bpZQnunFfZsMh29dwGGzc9DwiFfnmbsOkcW/7UUplsSjhJuhBWwm7K+IeSc7XVUOjuCJMZbhCdFdEQyI4vokISqmJSqmFXBC3cCOjIFEbGSxSGBFlSJ5p3NTv5Pd2Y9d2RySwYcOGnP/4j/+Yefrkycyx48fX/vVf//XuRYsWnTWOS3LTdz6AyQDOCyGKHccpllI2RL/n5ILCjiy/mzwXkZ899egYtx4SetXDJV/7/FnpNYVGAjgrpSxmV1+8mkKDeZA+BXpX+kUjqvByjzkWwP8C8G8Auvx53lkmTpz4Tdd1rySimA7phRCHAEyK8u0p1POIe4vFYkk6QqGQfOmll+786KOPXiwrK5t11113HVq7du2G6dOnR/0MVkrl8CB9FoAmKWWelLIIQJodkncevj6d4rruEuhB21kp5VYWenbpPFA4BccvLvauG01xa63vuj3pa1ZDNLsYWvB3kvek7+mN4TOF0+f81+3+uPsObmbj9Z8QNSsRDXZdd7ZSaj708OwwD9IPJEJdFwQRpUdJ4MpBxwQuc0jerZr1W996bs7OnbsnB33v9ts/VbpmzZr9QPtn5Vyu5dOFNmsUSikrE/VcdhWKTBHxi4tNYX0LDHECwulzocvVrPxZOZl3pV8LAEKIUhYgH4nH5xb13JX+KHSP6lcxPzjma1/72q3vvfdeSVtb26pY37etmyyW/okdoFssFoulA3/+85+H/+u//uvzeXl5j6elpaU+/vjjJY8//vjmtLS09ov05ubmlNTU1ExvUEtE06AjtNN8d2cOyf0Nn4ZkbTjEC9Ix+TdwTGEWgEoepEc4Akjv14twRBhDclOZ7DUcgvbrJa2buDMYooQFADKEjn3eKqWs6unrzmw4oKMrwnTy1yHYFdHrcfc9QSk13HXd+UQ0B8Agdv/mx+JcdhUKjm40I9c9N7OC/sxpj1wXndhvGG/YiTaPG+ipQoh9UsqivjiX3cHnSvE3fMxmcwvCQ3J/w6fHzU5uCk3iptB1AMDncpuU8nAcm0JddaWPA/A4gNcBxGTdSRATJkx4Tik1mYiS1vFksVgsyUYoFBJ/93d/d/tHH3304r59++bccccdh9euXbthxowZ7YMC13Xhum6G4zjZXDeNhx78ZgXcpTkkN+umATUk7ywck7+Eh0NNvMZpmzDi0SmcABU0JPfvxfaLW73/JoS4NV7wNdVUXol1JXRSQoHjOMWih1HzpN20pkjBFLeawvr2QSEir9tre3oMvYkhml0I3Rs5x0leO/ri9wgQt5rn3+wZnEf0BK4+Sc0iolTXdWeyKGE0dB+jiMUySfGa4J5NzqX2wkOLFM75atYa47MnFscxxHXdGzkVLRvAGRYgl8TrXFLwrvTLudK/DC20/r/xOCYAePzxx//i97///TbXdb8Rr8ewWCz9CztAt1gsFktUXnjhhewPP/zwpdLS0odc1x00b968lsbGxtRjx46lDx8+XPz5z3/2btoKoJb/AdpxlwnghJTyI8dxDibDYCiRoI7u/noANdBNHv+OsYvwDcmTseEQL9hdcT27K0ZBuyu28m61qENsih75nY3IhkMjIhsOZtOtXzU7uZExi0UJIwGc4gbb7lj/rgENB7PhY0Y3miIF89+5RG52ElE6i2XmQb/Hz3BTqCQRxC1ElHaJho8nlOqwF95o+PSaI4tdP15TKAc6UcBrCsVs15/vMQdROKrQc6V7TSFzbcR4AF8F8DMAp+NxLAAwYcKEbyulxhFR3JM2LBaLxRLJN7/5TXHo0KFlO3fu/F5VVdV1M2bMaBw5cmRKdXV12tGjR1Nef/11fPKTn/RuXg8gBD2wyoT+O9Em9OqhrfH6u9WfcV03Wyl1KwvqCMAp6AHNCOghuZkAdS7KkPyyKTgDAY4kX8RCT1fo9Kn8S6VPsUjBE1cGrakyRQpevZow4tZ4wQKPhaTXN7hCr2or4N83ZlDkmrCgBK4gN7M/gas5lscUSyicijafxTJKCLHLcZzCRIjKp8gEtFzj/OdCpxF4mEkKZhJXbW+JFEivwriC12JdA30u90gpi6WUx3rJlQ4iOs0CZL8j+ysAzgJ4N+YHwjz22GOffP/997cqpZ6L12NYLJb+hR2gWywWyyUQQswF8AKAedAF+BEA/w3gx0R0yeaGEGIkgFcA3Am9X7cEwHeJ6MO4HnQPEUJkAHgHwHTo+CHBX28dNWqUM2nSJDFhwoTGmTNnVq5YsWJ7kJOci5wrXde9BVp5fYKj3RM2wqyvYFW4uRPeHNKaqnyCfi5aAFQJIfZLKc/wrvSE3K+XaBjuisXQzp96IUShlLISQEZAw8EseP2R36FkaDjEC+9cKqUWENFV0K/BYsdxiroSSW64UoIaPub5v4DIIbnXdEjqdQNAOIpUKTWPGxmtQq8dKIz32gEKR5eaDZ9cdBTpmCIRc8/eub5ypQTB59JrCs2Abgp5UYVH4+xKvxrAaOroSp8I4K8B/BTAmZgfADN+/PjvENEoIjp7+VtbLBZL7BmgddPT0J/xV8KInE5PT2+ZPn162ujRo1unTZt2avny5TtvuOGGI0FOcqVUhuu6i4loLt9nEQ/SG2Fph8JrkiLqJaNuSg34sTOcTnOUr9mTKgGqL+HX5Xx+XQ7i66ntAFq9uhXhIbl5/s3Ib/+aqvqB2AtQSmUopeby+qGhAMp5kF7RlfNBRIONBK5cRJ5/UyTSQaTQ2+LWeMHncjafy2EAjrLDf1+8axIiGqSUihiQG3Wrl4DmiUQixMXcN0gocRSfS0+AnAktjN/Gwvi49Dc64Up/DNo08ut4PD4AfOUrX/nUBx98sJGIvhOvx7BYLP0LO0C3WCyWKAghrgdQBGAfgJehnQK3QDeGfkdEn7vEz6YBKIYeAH0Lumm+AsBdAD5FRJvie/TdR+iq6r8BHAZwEEA5//dESUlJ+g9/+MOVW7ZsWXnhwoWsRx99dM/TTz+9MSMjI7DBQ+EddbdAXySfllJuYufvgPkDxENCT5XsH5KbqvCgnfDtqnyl1Hil1BIuOs7zjrrigTjA7Qo+Vb53/scCGIPIuHVzv6HfFZF0IoX8/PysTZs2jR03btz5L37xi0dTUlLi8p5TSmVzg+0mACncYCtwHOcY0GHHm7/hY7pS2hAc3ZhwDYd4wVH5c0hH5Q8FUMWD9AM9afhywy2o4WNG3nuvf6/hU2Oc/6RLsiCioexKnwv9OXuam0K74tgUGsQiiKlgV7rrunUAFgP4CXRDKC6MGzfuBehBQl28HsNisViiMYDrpkcALIWulby6qbKmpubiv/zLv8z/wx/+8FJJSclfLF269MTq1as3LViwoDzafRHRkLa2tkVENB+AZGHilq4IE/sDfM0SbUhuXrfXw6iXDCfnWQApvMZpEYChfG26xXGcqDvqLRoiSide0QYgRymVC2ACtCjGLIa88x+0piphxJWd5fjx4+lHjhwZsmjRorgJESmcirYAOrEvxDVTe/oURa5p84uLBxt3Z55/fwJXvxeJkI7Kv9p13fnQfaZG/swsllLW9+B+HSLK8omLPTf5EOOm5vk3xcVJl2RB2pU+jQXIV0Enoux2HGeblPJEnB4T0K70qxBei3VaKXUVgGMAfhOPxwWARx999LY//vGPHxHRC/F6DIvF0r+wA3SLxWKJghDiBwCeB3AlER0yvv6/ofeZRm1UCyGeAvCvABYRUSF/zYF2UzQQ0aJ4H3882bFjR/qPf/zjJ7du3bq6vr5+5MMPP1y6cuXKDZmZmVEbPK7rTuRB+pXQxeImVrf2iwKPOkanear8bARH13lDcv9+vcs+llIql50qNwBoYadKwUB2qhiuFP/5D1Lln/XOP3R85mgimg4gnRtsW5O5wdbQ0OB84fOfv6do+/aZ3tdGZWWdfe1nP/uf2267LW4OWKXUMCPycSi0irwZutlgRn570Zn+hs+AdKUEwQ22GbzzbyKAeo4kL472PqfIHYd+N7nZcPMi783owJr+ev65KTSVm0JXQzeF9rArvTpWwhgy3PzcbJ7MQp0MAI4QYjuA3URUFZMHNFBKYcKECS8CGEZE52N9/xaLxXI5bN0UnZdeemnO+++//9L27dtvW7BgwalVq1Ztvvnmm8ugr4k6QESDXNddoJRaCCBV6NjnzZeK0E42iCidr9nb6yXjmt28ZjH3MpuR651ak0REKbwuZwl0PVbJooTKZBPGxhLSa6q8IaF/TVWGcdP2yGnoumkQEU2DFiHXspi7RCR5GhQALFy45JGjR4+OLyzMe3X8+PFxFacrpYRS6lq+zp8E3R84D90vCFrT5h/U1vaHcx4reO3APO6NpAqd1FckpTwU9D43xN1B4uIsdIy8r/HeA1LKGj7/fb5uKx4opYYrpW5SSs2GFrVVcw26O5avOdJufq9mHQPgCiLKgu7H1ALYSkSliL4rvds8/PDDt3344Yd/IqLvx/q+LRZL/8QO0C0WiyUKQojvAvgb6D09tcbX/x7AGgDDKUocoRDiD9CRrtf6vv4ctCtjAhHFRc3ZmxQVFaWtW7fuq1u2bPlGTU3N2Iceemj/M88883FOTk5UB5zruuOUUrfwIOMcD9JLkkGpzsXWMAoekptOzoghra/pE7MhlTGwnAPtVNnhOE6elLLfRvgaDTdvt177nj1EulLMvdh+V0QH0Qbp3d43slMlC8BhPpdlyTZU/OIXv3j7pg8/XPAakfg8gP0AnhSCjg0d2rhj9+5/HjZsWLffaz5Vvt8VYTbcPMfyIADNQogyKWWhlPJEMrzXEwml1BhuCs2CbqyVSSkPEZGCdkN4z4GZZuGtHDAdEQO+4aaUGsZNoTnQTaETRlOoU40w/gwyBQpB8Y1mmsJZIUSqEGKQEGIoRd+V3m1aWlrEFVdc8T1o1/uASGuwWCyJha2bLs+PfvSjWe+9995LxcXFd8yePfvMqlWr8j7xiU+UIvogPd113Xl8bTpI6J2/m3h9U8JDRGl8ne6J+8wh+VDjpo3ouI/cG5LHZEhFRMJ13WuVUkuhh78nWJSwL9mu8zsLRSag+a/ZzTVh7XuxEZkAVSuiJBC5rjteKbWYdCraRRZzFyarmPvNN9+c8NRTTz0GAJ/73Oc++vnPf76xp/cZIO72J3CZkd/N0DWTBHCCa/q9QoikS0DrS4gojQUz8wCMBFAjhNjP/82kSHGxKe4+a7z+zdop6SPvuwtph/+VLECeDl3P72KHf6f2zlPkqrZcT7AAXb+afwM8N3+NEMIVQjj8fHlrsQ5Qx13p3eaBBx74yw0bNrxPRC/F6j4tFkv/xg7QLRaLJQpCiAkAtgHYAOA56DjBvwDw/wH4JRGtvsTPVgPYSET3+77+WQC/BXA7Ef0pTofe64RCodRnn3320c2bN3/z5MmTk+6///79q1ev/njUqFFRnRJKqdGu697CzbJ6jtXb3hk3QTwxFMntzR5EDsnN/W7nEBkd6P23LmhIG8djHswNtgUABgsh9nJTqFPFTaLBroig6Lqghpupyjefh269jki7Va9xXXcRtPM3JKXMTxZ3RV1dXcq1V1/9rWfb2lLMinAfgGsBPPvss29/61vf2nup+yDtpDWFItGGtNEabiHBEdlKqVGu6y7g4a8QOg4uP1lfm70FN509J7P3PIyEbjg4xk3PQzfazhjnvwY6xrAvDj0p4Pf5dKXUHG4KtRpNoZMUdpO3D8qNhpspFGmAbsp5TZ8a428A+R4TAMaRjngfw4/h7Uqv6snv09jY6EyfPv27ANKJqF86YiwWS2Jj66bO88orr8z47W9/+7fFxcV3z5w5s/aZZ57J+/SnP70H0Qfpqa7rzmEX9VAhxB4epMctVaizGNfs/iG5/+/lBRhDcnD9ZF4z9tLxQik11XXdpQCmQLuotySLmNsP163DKXJFWNCaJC8BLWhNVbevGZVSma7rLiSi2dBi7l0sQO7z12ZXWLhwySMVFfIKok/KtLT1zUVFnXehG9fs7ec+irjb6xv4/9UJIYjf57O4nh8JvY8633GcPX3dH0lk+Hp6hO+aPQfAKET2Ddqgr9lP+MTFZ5Pxvd+b8Pt8Nul1bRkAjvFarL1CiDaK3A3vFxd7n0GtMCLvAXh74UNBQikiGkzhXelDiOiCIUBu6snv84UvfOHTmzdv/h0R/V1P7sdisQwc7ADdYrFYLoEQ4hro/TtX8pcIwGuXagLxzzUD+AURPeX7+kIAWwE8QERvxOGQ+5RQKJTy/PPPP7Rx48bnjh8/PnX58uVla9as+Wjs2LFRnRIcR34zEc0E0Mjx2dviOazkv31DjP165pDcVCQDYSezf0iecPvdKOyiXgwgUwhRLqXcLKU8nGjDNJ8i2d9sGG7c1HPSmoPaXmm4ua47QSm1KJncFSUlJcM/9alPrf4dgDt93xshBN2xfPmff/rTn24B2ndMRpx74znwR94HRa53auWA91iu687mqMLh0A7/fN7tPSAvRgPeA2azwYxvbER4SBuC3qM9VCl1DYCroB0BOxzHKUoWV1qiQNrdN5ljCqdAf/a3QTd7vIaP5yZv3w3P8Y2BDZ9OPq63K30KdFOomYgOK6VK0Q1X+rlz51Kuvfba7wBIIaKE+rtksVgGDrZu6hqvvfbale+8885LRUVF915zzTXnVq5cWXDnnXfugr726gDpOPKb2EU9XAixz3GcjVLKk/E8TgqnDwUNyf3X7N6Qtr1u4mv2hEtH4VS0pXydf57jyIt7c6DfGYy6NdqaKlPc7U9A65U1SaTXDszh4e8wrkG3RovQTiTC7vM3ASyGEFPp3nvv/Nh0ofM1e7QELvOavT3yXkSuSjrb2SG4J/JQSi1kkWeTEGIb76OOui6vv+OrWz2BcS4iUwBdhMXdXuz6BSKaTDrefSiAKq6Z9vem4SHZIe1Iz1ZK3cCfmTnQf6tchD+DAO0m94uLa7hv0OXPIP78m8CPOZJi4Eq/7777bt+6deu7RPT33fl5i8Uy8LADdIvFYomCEOIqAB8BqADwT9BDiwUAXgDwNhF99RI/OyAbQR6hUMh54YUXlm/YsOE7VVVVVy1btqx87dq1H0+cODGqGl0ple267lIubi5yE6OoJ00MVsO2xwUaxW42dEyaRwOCh+SdLnYTCSKSrutexw22UQCOsSO9V+PIKbwXPsgVkYnIAVU0V0SfR9cppbLYXXETtLuihN0VNX16YAE0NjY6M6ZPf/aJlpb0fzK+XgxgLoB169YdWrZsWQp00TvEuIn3HvA3fAIj77sLF9/XuK67ENrhf05KWeg4zg4RJSYy2SEir+mZa7wHvIaPf0jb3vDx3gOXOi/sCJjLzp/BACq4KZR0qwfiBUW6ySPcEYh0pjRANz4HQUebtgoh9vLfotNxOjYg7EofzQ3aGtKu9EOX+fF2zpw5k3bDDTc8D0CSLe4sFksfYOum7vOzn/3sirfffvtvCgsLl0+bNq1hxYoVRcuWLduO6IN0h2OKlwLIEnpNzkbHcY539xgMUZ+ZAOUJi0cgnD7Uio5D8hAL+JIy/UYpleO67hKuQVtZMJvf24JZ6uhkNoUKZt3qxR2bK6oSQtzNr83rWMw9GsBJvo7a29fHFo2w+3yP1JflKzF48Pq2bdsKdmRnZ2dSx73Y7U5adEzgimktw/2RBUR0I4AUvi4t6Ml7PZExxDp+N7m/bjWHtGbt2iEByrxv13VncLz7JAANUspi/nc+/r9dckBhN3l73HpA3doKnaggoP8+pEKvxSrgxIS4vNcp0pU+mLQr/TB10ZX+uc997o6CgoJB8lWSAAAgAElEQVS3iejH8ThOi8XS/7ADdIvFYomCEOJXABYDmEJEF42vfxnAvwO4lYg2RfnZARVFGI1QKCRffPHFezdu3PhCeXn5dffcc0/F2rVrN0yZMiWqU0IpNYKbGLMBtPKFeEE05wI3G7K52eAfkpuFlhf3HbRjL+GjubsDK9inu667BMBk6KisLbzvNyaFDXXc7+Z3RfidzEGuiG4pknsbdlfMZXdFhhDioJQyr6/dFRSOrsshopwXXnjhhl/8/OfjvgfgPugd6M86DuSECfThxo3VaWlp7U0fbvbEbMdkV2DnzwIiuh6AK4TY6ThOAQsnkgrquHbAbPgMNm5ah+CGT4+cQaRdadexw38cgDopZRELE3oUc5cscMMnIm49wJliuslruOHZwU3OopnZ/HdoCIAjHFW4L16iKiJK56bQFABDiegiaVf6Puio/qhUV1enz50795tE5FzqdhaLxRIvbN3Uc15//fUJb7/99vcLCwsfmDhxYtOKFSuK7rvvvmIpZeA1Owtmr1dK3QJ9vVHhOM5Gx3ECXXm+68UIgTEiha1m3Lc5pK0VXUgfSjaUUsNYMDsXgCN0us9WKeXZWD0GXy/6ncyeUMGMvDedzElXt3INOsV13cXQiRQNXNMX97Vg1rhezPnoo4+mPPDAAzcCrwBYxreoBnAr1qxZdeHZZ589hmA3f28fczqnT8yHHuYf4/NZmmwuatIOYm9VmF9c7F8V1u5kRrhuDfX0PcCrBOeRXjHmCCH2SSkLpZRH+uvnm4khmAoSFwcJFfyO8vbejSFMmAM92G7imr44XsloFHalXwNgFHXRlX7PPfd8pqio6P8Q0avxOD6LxdL/sAN0i8ViiYIQYh+AaiL6lO/r1wHYDWAFEf0sys9+AH1Rd53v688BeBnAeCKKa9xeIhEKheRLL71058cff/zigQMHZt11112H1q5du2H69OnV0X6GmxiLuYmhhBB7hBBHAbTvJ4cejPj367U3GaALrVr+/wkVx9fbuK47USm1hIiuht45n8c75zs1OCWidK/ZgI6uCHO/mz/yPi5O5r6EC8Xr2V0xCtpdsZXdFXH5HQOECqabPyK6znXd2u98+9vO//nv/85qcV0BANMmT67+yeuvvzl37ty6eBxfT1BKZSil5iql5kI7gst5kF6RSMIKCu+ZzDGaDTnQzYZM46bNiIyuM98HcU+0YGHCfBYmQAixh4fpSe9WocjY+wh3BCLd5J47y9/06ZJQgd/r1/Br8woAF4ymUFyEHtTRlS6IKESXcKUfPnx48OLFi9copVKDvm+xWCzxxtZNsWP9+vVj3njjjRcLCgoeHj16dMuTTz657cEHHyy8xCBduK57LQ/SRwE4JoQohR5AmUPyLISvF/0reswBbVzjvhMdFszOU0otBDBYCLHXcZwtnY3K52uVdmGrT9xtXi+2oKOTuTYeTua+RCk10nXdRTysdFmYkC+lPBevxyTtZM4OeA5yYFwvfuc732ldv3594LVTbu7omv379/4kXsfYHYhIKKWucl13AbTgsoGv8YsTTTBLHRMVTHGxty7PWzvgrUkya6fz8R5m83v9Bnal50DvnS9is0Gvi8tjDYVj74PExf7d5DWGUMF7DrokVODVjHNIJyYMAlDJNVPc4vJJu9LNtViX3ZV+1113fXb79u3/RUSvxeOYLBZL/8MO0C0WiyUKQog/ApgJYCoRNRlffxzA/wZwLxH9NsrPfg3ATwAsJKIi/poDoARAHREtiffxJyKhUEj86Ec/+vSf//zn7+/bt2/OHXfccXjt2rUbZsyYcayuri6tpKRkcnNzc+4nPvEJ4RvQmrvFXABnhBBnRNjN3O+aDfGCmxhLSO+cbxFCFPBe7ybq6KI1XRHmcMpz8we5IpIu8r67sLtiKrsrpkELEzx3RbcEG0SU6ms2mA0fM77REyr4h7Tt0XVVVVWDN23aNGrSpEnnb7311oR3dRvChAUAxkInJhQ6jlPSm00MUyziDWeN58BrsnmN54iGDw9VEyLClIiGsFtlLnTDtpqbQnsS/X1K4fjAdkdEgFikPUKTn4caw5kS89cLR7x6TaHBAKrYlb5fxC+qMMiVfoR3pbe70isrK4fcfPPNzyil0qPemcViscQRWzfFnrfeemvk+vXrXygoKHg0OzvbfeKJJ7Y/+uij+UTUVlZWNrq0tHTyzTffTFlZWSMQHpLnIPx3EtDrSU4FXK/X9Rdha7yg8M75xdDXURW8EquKvx9tTVUWgt38/top7gPCREIplcGu33kABgkhSlnQ3S2BJ4VXhfmFrf61A6aT2UzgCp08eVK8//7744Luf9asWbVz5sxJOOGxh1JqFMe7zwIAIcRuFibEZe1QEBRekxQkLjb7N2bvwL8bvs+j/VmYMIUFyFdB90h28lqshK6hWbCTZTwHprjYdJN7vYMgcXGsjymFRV1zoVe2NQohtjuOsz1ewhmK7kovI6LD5m0/+9nP3rlz585fElFCCWQsFkviYgfoFovFEgWhYwN/A6AQwKvQu/wWAXgOQBWA2UTUJoT4BYAvQTeMjvLPpkGvHR4G4HkApwGsAHAPwsVcEAuJqDDK8TwKYH3AtwjAWCLqtWKpuwghUgFMkVJOHz9+/GfPnz//ubq6ujG5ubk4c+aMICLMnDkTH3zwgblbzGs6nCeiqUQ0B3oH2HZ2AyRsYZuIcJGVqZSaqJS6CbqoEdDDqDTjps0wGj2IFCoMaDd/ENzEWMTCBJdfnwVBRSJ1dNGaDR+z2eCPbzSbbgk9AO0JLEyYxPHuM6CbGNsdxymMVdEd5TnIRccIzfPo2PCpEUmUqMBNoemu686DjtG8wO6foni6fzpxXGZ8YIQ7Ah3d5P7owB7H3vfguL2m0BzoHYqNfD63xzLm1feYgHalXw399x4UdqVXlZWVDfnkJz/5ddd1B1/mriwWiyUu2Loptgg90cgFMD07O/vG9PT0L4VCodlDhgxxWltbZWOjXs/95ptvYunSpecQjlqvBRACkGFc559wHGejlPJAX/zdTFb4b+8Q13VziWgmX5MOha6ZBMJrqjwXbdCaqgHt5g+CiFJd172RHf7ZAI44jpMX9Pr0noNLJHCZq8I6CBWklDVIEGFrvGDB7GyOdx8GoIoH6WWxeu2RdjJHDMgDhK1t0M9BB3FxMpkceJ3gXAqvcarkGjRm57M7+J4Dv7jYdJObsfemuLhPVj9wj2QuCz3ShRDlUsptUsqD8aqliWiQIUAeQkQXDVd64x133HHXrl27fkFRUnEsFovFjx2gWywWyyUQQiwF8G0As6CV50ehm0M/IqKzfJv1AB6BbgQdMX52JIB/AHAX9MX3TgDrAFQEPNTvoIdlUyjKBzM3gv4dwJcBHPB9u5iI+ly9eymEENdDnwOvyLoIoDwjI6P22muvvdZxnOxx48adXbZs2dZPf/rT26MVuqR3gM1XSi2CvggvcRxnU7wGF8mIocgPckWYOw7bAHjnzYt19NwVh/tzsyFesLtiPrsr0gEcFnr1QAqCnSneTmZ/wyckhLjQJ79EAsFNjHksnEkXQhyQUuZ35vVpNN0iBuQBzQZzL3b7oLw/ikWUUtncxLgJ2v1zkHf+xS0un8Ju8nZHBF06PjAE3fDpVnxgb8KJHnOJ6Abo93uF4zjbuMkWr6ZQOrsrpoJd6YcOHTq1ZMmShW1tbUMv9/MWi8USL2zdFDuEEL8BcLfxpeOO41ROmDAhZ8qUKVcSES1atOjgV7/61fczMzMbg+6Dwnuob4FeQ3JaSrmJVw7ZRiBDkVHTEbUTItOf6qFfdxn8r14IUcSC2YS9VklUWOB5teu6i6AFiXVCiAro85xluGj9z0HQiqQBn6hARJJ3US8EMAHAOU5G29GZeoYiY+/94mJToHnZFLT+gCGYnQ9gPIA6Tp7aLuIUl09hN3mQuNjvJo8mLo7HofUYFs5cz670cdDrB7bzv/o4PSagXelXQ6/FAhGd+au/+qvcoqKif1RK/Vs8HtdisfQ/7ADdYrFY+hghxK0APgLwt0T0/UvczmsEzSOi7b10eDFDCDEUumFWBuAggONEpAAd7b5u3bp5v//9739QUlLyF0uWLDmxevXqzQsXLjwY7f6IKM113TlKqSUAhggdW7aJleb9Horcid0e2xhFkR/NFdHgFbpc1MzmmMLhQogyKeVmx3GO9sGvlzRQx7hvs+lmOvrbAJwBcNhQgkc8B5bo8OtzFjeFcqF31OU7jrOHv59tNBvMho/ZdDuHYDf5gHsO+HzO5J1/YwDUclNoR3dcIj43eYQ7ApENH89N7m/6JPVzwOfzOm4KjQfQYLjSY5qS0tLSIoqLizOLi4tzjx49OvXUqVOTKioqsvbv3z9IKSUvfw8Wi8WSvAyguukOaLfzQQAVRNQ+JM/Pz8/4h3/4h2fz8/OfBDDoscce2/Xkk09uSk9Pjzokc113ouu6t0KvHArxIH33QBk6kl5TlWVcJ5prqszkITP9qda4Xq81BX2u605QSi3lwUyDlHIrD9aSfm9yvKDI3fD+BK7hvpvXAzgqpTwV7TmwRMd13XFKqYVEdB2ANqHjyAuFECEiGuY7/57ANRPhxA8vic4Ttnp7sQfkc6CUGssC+euhA0L2SimLpJTHujOwNtzkfnGxKbJvQbC4OOmfAz6fc9iVnsJ9p23xEHRXVVUNLiwszNm3b9/Y06dPT6+urh65bdu2Yc3Nzb90XfersXwsi8XSf7EDdIvFYuljhBD/CeBB+JwYAbdL6kZQZ3n55Zdnv/feey/t2LHjLxcsWHBy1apVW26++eYy6Hi8DrA6eLZSaimAYUKIvTxIP9W7Rx4fzAEtOroizH23nhrcbPaERBejplm9PpOFCSOhY/U2c8xWTH+3ZIEV+VlRGj6m49OM+25fQUBEI3nwOxF6UJnHe72TuvjtTThVYRg3G64mvZ8uE/pzwXxhXkSkM8X7b7+Ove8u7E6bqJSax002xWKkQinlyYDbD+bPo3ZHxGXc5DUIN90S2k0eK5RSY4ymUCq7/Iv5M7TThdepU6fS8vLycnfv3p178ODBnMrKyhEnT57MqaurG+04TmNKSspB13X3tLa27h03bly14zgNR48efS+aG9NisVj6A7ZuCrNhw4Yh69atW5OXl7eira0t4ytf+cqeFStWbBw8eHDUBCMerN3Cg99zLJbdKRJgD3FP8Q1oTXFrNvQ1o4c3mDLTn7w1VV0SEXISzWL+m98ihChMSUkpiJdDNdFhh2eG7/xHS3/qELkOoFUpdSPp+GwphNjF8e5n+uY3Sk68VAXXdcfz0HcCdNKcWTd5InvvOfASuPp97H13IaLBruvexILZLAAnpJRFLEZq893WdJP7xcV+R380cXHv/XJ9AOlkyZl8PkdDu/yLpZQ7pJTnO3s/LS0tYvv27Znbt2/P3b9/f25FRUXmkSNHMkOh0OjW1tahaWlpVUKI/c3NzbsAHJg1a5YqKSn5kIiq4/bLWSyWfoUdoFssFksfIoQYDuAEgE1EdMdlbuvt8jsFPdisA/AxgO8R0d44H2qv8/LLL898//33f1BcXHzHTTfdVPPMM89s/dSnPlWK6IN0h/epLQWQKXTU80bHcRL+wphdEdkU7IowB7SNiO6KiOlwkMJ7k2+GLrpPSym3OI6zpz+6VSgce+9vuvkV+e1NN7Pp05m4b3arLCK9k+uiEKIoJSWlUAgRGL05EDEEI54i3xzQpvLNFIBaaHfKYOjPQ0cIsZ9fo8f75OCTHCIaymKkudBOoBohxAnSMbfZ6Ogm90Q7/qZPwsYH9iakU1K8ptCY6urq+n/7t3+reeSRR/44Y8aMkwCglEJJScnwbdu25e7fvz+3vLw868iRI5lnzpwZefHixcy0tLRjUsr9ra2te1zX3QdgP4ADRDQgklYsFovFxNZNwezatWvQD3/4w5VbtmxZefHixcwvfelLe59++umNGRkZUa8vlVKjXde9hYiuBVDP10/bE11syAPaYVEGtKZ700XAgJZrpvOxvk5RSg13XXcR6ZVDQgixnQe/52L6QAkCRcbe++smU+DtpT/5I9cvuRueiAZx0twCaIF8uZRyq5TykL3G1BiCETPu23sOhhk3bQQQgu6hZPP3zrKge+dAELfGGu6TXKmUmk9EV0I79o9CryHIuIyb3C8uTujP3N6ABd3jlVJzWfAhf/rTn54eO3bs7mXLluWlpqYSAJw+fTotLy8vh8XFuYcOHRp+4sSJnLq6ujFCiKbU1NRypdTelpaWPdA1034Ah4howJ9ji8XSM+wA3WKxWPoQIcTXAPwEwP1E9NZlbns7gCUA8qEHRzMBPA99cb6IiPbE+XD7hFdfffWad99996Vt27bdff3119c+88wzebfffvseRB+kew7qm6ELyHLHcTb2dRQ5hSOOg1wRI4ybepFp7Q7mzg5o43TcUEpN4pjC6dDK4K0c9Zx0BbfhoA2KXPfH3ocQ3i/m/etx000pleW67oKB6q4wFPk5AQ0fM0bzPCLPv/e/I1IVWL1+E++oywJwjHf+lfZHsUesoHB8YLsjIsAhBOj3Q0gIUS6EOG6sIEi6939fQERoaGiY+Ktf/eovv/e9701saWnB2LFjz9fV1TU2NjaOAOCmpaVVEFFpc3PzboQbPhVE1Ouf+RaLxZKo2Lrp0uzYsSP9xz/+8ZNbt25dXV9fP/Lhhx8uXbly5YbMzMyGaD/DDuqlRDQTQCNf4xeLPowi5x7lEKVUdsD1uimoJADnAobklx3QxvHYB7uuO58Hv4M41WeLlPJ0bx9LTyGdwJVpXqtHuV43Bd7mv7M9HQ6yQP46XjE2GsBJHvzu7Q+pCZ2BiIZEERdnQbvKAe3o93oH7bWTP1WB6/rJSqkFRHQNgGYWexTGeuVQf4JrV3M/fHv0OiLd5IBeA3GURR/murDeP/AkxHXdQSdOnJj/4IMP3lJaWuqMGTNGNTU11bS0tKRcvHhxRFpaWjWLi3cb4uL9VlxssVjiiR2gWywWSx8ihCgCMBnAeCLq8jBCCDEZwG4AfyKiZbE+vkTin//5n6f9+te/fqmoqOhz11xzTd3Xv/71/LvvvnsX9HCnA0QkuOC+Bdp5UuU4zgYpZVW8Chh2MUdzRWTiMrF1UspaJHBkGrtVlrAy+AIPKYuEEFGjIvsCn6Pf3/Axi9x6BDd8uhR734PjHOS67lwe/A4TOuo5rz+4K/j6cuglGj7+uO+ghk+XhofsBrjKdd0FAKZA76Qs4kbwQI3SNOMDI6LX0dFN7o8ODBFRGse73wgdR35ASlnYH16jsUYphfLy8qEFBQW5paWluRUVFdmHDh0acfr06ZGNjY25qampp7Kzsw/NmDFj5IYNG4YS0TgAhwH8DMB6Ikq65rrFYrH0JrZu6hxFRUVp69at++qWLVu+UVNTM/ahhx7a/8wzz3yck5MTdUCmlMrmQfoNAJp5SFkYT/Guz8XsX1M1yLipd70etKYqIYeoRJTKqT6LAIzga/zNjuNEXTvQF7CjPyKBC5GOfu9iz7xebxe2SilrRRdj77t7nEqpKa7rLgZwJfQ1fgFf48f98eMNdUyjM8XFQXHffnFxlwUjSqlM13XnsaA7Xegkr3wp5ZGBeo3PYoUgcbHfTd6hZgJQp5SaoZSaB2AstMt/G5sOEqpXkgjU1dWlFBQUZJeUlOSWlZXlVlZWDj9+/HjO2bNnRxMRpaamVtxwww3NR44cyTx58uRk/rH/C+Cn0Ck0dphlsVh6DTtAt1gslj5CCDETQAmAV4lobQ/u5/8BuJGb8f2edevWXfHOO+/8TVFR0fKpU6c2rFixomjZsmXbcYlBulLqGtd1bwEwBsBRx3E2SinLu1Mc+gaD2QGuCNPFHM0V0dAXrohYwQX3Ii64SQhRzA7q+t46BhYrBDn6cxDp6Pd2Yrc3fTgyrTZRHLTsrriem2yeu2IruysS2kHNDZ+cKA0fs/l5DtEbPjE/LqXUKHb5zwIAdgDlJ6MDqDMYbvJ2R0QUsYLZ6Knp7HuBdBz5LBZ7jOSfLXIcp6QvkjH6kqamJllUVJS1Y8eO3LKystyKiorMY8eOZdXW1o5ua2tLS0tLqwCwj93kB6CdEWVE1B6lK/SLfhGAJwB8EcB3iejHffH7WCwWSzJg66auEwqFUp999tlHN2/e/M2TJ09Ouv/++/evXr3641GjRkWNFVdKjeBB+k0AWoUQBbzTu1sDIHYxm4PBbOM60XQxNyH6mqqEuF7vDqST0a7nFWMjoevQzVLKg71ZCxLRoIAELv+KJIJO4KpBcO3aW4d7STg1YRFf47tCiB18jZ/Qcfk+sYIp8M6FFtp7NCNyQGs+DzGPomaxxw2cmpALXYfm89q2hBSo9AT+TIomLjbFCmbtaq6ruuR7gcUeE1iAfB10r2QPu/xPxPe3SyyUUqisrBxSUFCQu3fv3tzy8vLsqqqqzNOnT+eeP39+ZGpq6mnHccra2tp2t7W1tbvJAVSbA3IhRA6AR6HrpqEAriAby26xWHoRO0C3WCyWPkII8S8Avg5gJhGV9uB+BkwjyGT9+vXj/+d//uf7hYWFD0yYMOHCU089VbR8+fJiKWVgocfFzHTXdW8FMB5ANQ/SDwQVQRTexRzkijB3u3l7gINcEQk9/OwpRDS0ra1tPhHNB5AmdBT5FillTCK0fGKFoAhHL7bO23MY0fCRUoYANCVKw+dy8Gt0KrsrpkHvpPTcFX02pKTI/fCmKj9IrOBv+NSIGEQ49uDYh7ADaD70zr8qbrKVJZuIxecmj3BHINid4m/69Lj5SeHox/mkox9dIUSJ4zhF/U2ccOzYsUF5eXm5e/bsya2oqMiurKwccfLkydyGhobRjuOcTUlJOei67t7W1ta9CDd8jhBRlz73hRDZANqIqNcESBaLxZJs2Lqp+4RCoZTnn3/+oY0bNz53/PjxqcuXLy9bs2bNR2PHjq2N9jNKqWGu6y4morkAlBCiKCUlJU8I0WGvOoV3MZuCymzo60RzMOjtAfbExeaaqqR3EV8KFnRPd113KYCJAM6wI31PrOpFFrWag0GzdjVThxoQPXI9aWpXpVQGO6jnQcfl72MB8vG+PC7S6U0RA3LjefDECt66MLN2reHatU/S6Pg1OlUptZD0Xu9GIcQ2x3G2SSnP9/oB9QAKr4GIJi72TrDnJvfXTLWxqF2JaCivGZsLXTMfl1IW9rcVBBcvXpSFhYWeuDjHExeHQqHRbW1t6WlpaZVCiAMXL17chXDNVEZEXXpdsQB5IhElVJKHxWLp/9gBusVisfQBQog0ANXQF46Le3A/U6HdGB8Q0X2xOr5kYv369WPeeOONFwsKCh4ePXp0y5NPPrntwQcfLLzMIH0qO9InQxetBwBcAA/KoQvcocaPmbvd/K6IAa9+ZXfqHHZQDxM6Am6L4zjHOvvzRJStlMoFD8ijiBXOIbjhU5dsw9DLwQ7qRaR3UrpC76criKe7gkUjEQNy43kwkxX8YoUabn4mbEw6O4BmKKUWApgAHatXyLF6CeWgpshdh97z4HeTe03oGu+5YDd5XNwpQSilhiml5iil5kC7yA6zK31fsjRg29raxI4dO0YUFxfn7t+/P7eioiLzyJEjWTU1NaNaWloyUlNTj0gp97e0tOxWSnnOiANElNAuJ4vFYulP2LopNoRCIee73/3uFzZu3Pjtqqqqq5YtW1a+du3ajydOnHgm2s+wWHYRDymlEKIMwHHov/tm1Ld3feKJWv0JXLVCiPPJImqNJ67rTlJKLSWi6QDqOC5/e2ec9ixq9YsVvGt1v4s5WgJXn+23jwfsoL6Rr/GzARzhZLQD8aoPWTQStB8+F5HJCudh1EvJIlZQSuVwkteNABwhxB4WdVf39bGZ+NzkfnFxRBIa10v+QXmvfCYRkWQBzXwAU6EF/ttZnJA0u+ePHz+enp+fn7tnz57c8vLynMrKyuEnT57Mra+vH+04Tj2Li/cEiIv7jVjAYrEMTOwA3WKxWPoAIcQXAfwPgK8S0b8HfP8XAL4EYCoRHeWv/QHARwD2QBdjswA8Cz3oXdITN0Z/4K233hr5y1/+8rv5+flfzsrKcp944okdX/7yl/MuXryIXbt2TTxw4MCku+66q2348OGZhoPZdM8S9ID2GKu/TVdEQg3YEhXSUeSzlFJLoBs5hzmmsByA554NavgMM+7GjHA0Gw595mLuS9hdMZ8bl+lCiFJ2V3SrgWG4mIPc5KZoxHSnmM2GukRu+HQG13XHsbviOgBtQoidHKsX6q1j4Och23ge2t0R6Bgf6I8OjImbPFYY4oR50KKk81LKYv7X0NfHBwA1NTWp+fn5Obt27co9ePBgbmVl5Yjq6ursurq6MUKI5tTU1INKqb0tLS17EG74HKJu7Ni1WCwWS2yxdVNsCYVC8sUXX7x348aNL5SXl193zz33VKxdu3bDpEmTTh45ciRn165dk4cPH56xdOnSFEQmP6Uad9MM4IQQ4pTvmr3Lu5gHKiyWXcJi2Ys8oCyEFnV7YsqgyHUvgStI1BrqSxdzX8IO6qtd110EYBKAWo4i39kZcULA/QFhF7PfTZ6NsGikDXytjo7i4qTuIRDRIHZQz4cWaBzjc9prYlmKTKQLEhd7L3RPNOKvmRLK8MDihHksTkgTQpRJKQullIcS4bOzra1NlJSUDN+2bZsnLs46fPhwJouLh19CXHy2r4/dYrFY4oUdoFssFksfIIT4AMBCAOPI2ItqfH89gEegG0FH+Gv/CODT0LFvgwGcBvBnAD8A8B3ovUDRWEhEhZc4npEAXgFwJ3TEWwn0TtYPu/7b9T5CCAkdy37ViBEjbhw0aNADtbW1s4YMGZLS2Ngo2tp0zfSb3/zGnTt3rhkdGBJC1AJIc113HoCroJ2pmxzH2dWforV6A9K73YZxgXsd6YjnDGg3ikS4wA1qNISklLWimzsW+zuGu2Uaw48AACAASURBVGIRdLPgiOM4W4OiyI1GQ5Cb3L8Tu31ADr3X2ns++pU7JQgj+nEugCFCiINSygIpZUWsmo7sJo9wRAQ8D158oL/p02tu8ljBzeB5RHQDtGNlPzeFDse7kauUQmlp6bCioqLc0tLS9obPmTNnRjY1NWWlpaVVSykPtLa27nFdtxTh/eRnyBZEFovFkrDEuG7KALDsEg/Xr2smABBCDAMw3XGcq0ePHn3n+fPnP9PQ0JA1bNgwUV+vt4nceeed+PnPf14PFhQbTuYGpdR0IloILewscRxnk5TSDk+6CBGlElG267qTSO/zHs/fUggPyYHwujD/v3OJMHBLRFzXHa+UWkxEMwBcFEIU8bqhDpHRpKPvTVGrKS6OcDEjWFzc0N+fB0OcsADAFdBrxop4zVhMand2k0cTF3vPA0G7yUPoGL2eVAkXpBP8ZrI4YRR0SkSR4zglohdWWtTW1qbm5eV54uKcysrK4dXV1Tnnzp0bI4RoTU1NPUhEe5ubm003eSUR9fsegcVisfixA3SLxWLpBwghpgAYGfCt30E7eqdEGxAIHYtYDGA4gG8BOANgBYC7AHyKiDbF5aBjiBCiBNpZAujh7KGUlJTKqVOnjhs3btxVbW1tdOutt+5/4oknfj9kyJCoF/1KqTGu697CxXYd76bbmWxDrHhDRIOUUtlGYWu6yT13CgE4C/36y4BWrZ8XQhRwFLl1dnYDo4GxGLopXAegQgjRACDLaPh40fdmoyGo4dMnv0ciQUQpruter5RaAGAMtJCggBsYnYnT9NzkuWbTB/p58LvJgwbl/e55IKJ0FnzMgz4Pp7kptKun4oyGhganoKAge+fOnbkHDx7MraioGHH8+PHs2traMUSE1NTUCgD7mpubdyPc8DlIRP16v6rFYrFYLo+tmcQjAP7T+FINgLLMzMzGG2+8cU5TU9OIqVOnnn700Uc/nDdvXnm0++Hhz1yl1GJoEeJuHqTXxPt3SCYoMurbn8A13LjpBei6yeHvOexM3eA4zok+OPR+gVIq03XdhUQ0G1q4ekgIUQ1gkPE8mNH3FxFZLyWki7kvUUqN5nj3WQBICLGLa/vTl/vZKCJvU1xsusmDaqZ+9zyQXi84SSk1n3tQLp/TIinlqZ7ct1IK+/fvzzDFxVVVVZmnT58e2dTUlJOamnrCcRxTXOzVTaetuNhisVjC2AG6xWKx9FOEELdCRxf+LRF9/xK3ewrAvwJY5DkuhBAOtKOigYgW9cLh9gghxL3QLs6DAKrM2N38/PyMV1555dm8vLwnAQx67LHHdj/55JMb09PTo0aqKaVGuq57MxFdDx1HvJUV1gNm6OtTgfsbPmbUt7nbzWw4nDMd/CxOWEpE1wJo4gFlUW8orJMZ6rjrMNd4Hkb4bl4P4IiU8pSv0WCTFDoBNzAmK6UWcHpCs9D76QqFEHWIjHE0Gz7ZCDd8PDe5v+nT7xo+nYHP6VSl1DwiuhpAixCipLm5uTgjI+OSjbZDhw4Nzs/Pz927d29ueXl5zqFDh4afOnVq5Pnz50elpqbWOI5T1tbWtrutrc1s+By3DR+LxWKxdIUBVjNNAbAEQBm0uKzdOR4KhcQrr7zyiT/96U9/s3fv3gW33Xbb0TVr1myaNWtWVbT7YxHibKXUUgDDhBB7eZDeo8FPMsEJXBn+eonCEdP+qO+I/fCc/nTBuL80PqeLoc/pARZ1H+v1Xy7JIKJ0I+rbH31vriBoBXASwFHeDR9i8UdTfxO1xgsiGuK67hwWyw4DcMhxnHwp5UHo1W3RxMV+N7mXSGfWT0nlJo8VSqkMpdQcpdQc6HN6BEBRS0vLgYyMjKh9qMbGRqegoCBr586duWVlZbkVFRWZx44dyzp79uwYpZRMTU2tBFAaIC62yX8Wi8XSCewA3WKxWPopQoj/BPAgjDjDKLf7A4AJPNg0v/4cgJf5e0mvfN+wYcOQn/zkJ6u3bt369ba2toyvfOUre1asWLFx8ODBUQsH3lG1lBXWF3mQXtRTB2WiwMPZ4QFD8lzo4ax/KNje9Olu1LdSKst13cVEdBO0wnobF9sJsSu5r2BXf7SGTwrfzAVHaSLy+WhVSt3ouStYtZ4npTzTN79N8sLCkSwepM8EMAHaDeQiMk7Ta/j4B+UDsuHTGZRSI1zXnUNEcx566KEhjY2NF+69997iq666aufu3btzDhw4kFtRUZF59OjRzFAoNKatrW1QWlraIQD7WlpadhOR1/ApI6IB/XlhsVgslthha6ZIQqGQePXVV5f88Y9//Ntdu3YtvfXWW4+vWbNm05w5cyqj/QwROZw8sxRAJg99NzqOU92Lhx5XiCidiLKVUu2rkQxRa5p3M0RGfXd7Pzyf05l8TnMAHHYcZ7OUsnwgX2uyqz8rirg4w7ipJ/Juv1YHcJaIxvNKrNEATnF9v9cKjruG5yZ3XXckEc0koqugz7+C7iH43eRB4mJ7zgMgIqmUusZ13fkbN26c/PTTT9Pdd9995DOf+cyfT506BRYXZx86dGjEqVOncs+fPz8qJSUl5DjOQUNc7K2qOkZEvbKv3mKxWPordoBusVgs/RAhxHAAJwBsIqI7LnPbagAbieh+39c/C+C3AG4noj/F7WB7mV27dg364Q9/uHLr1q0rm5qasr70pS/tWbly5caMjIwOOxU9OP5tKQ99W6SU+Y7jFCSLe5qIBgcMZz3nrDecVQgPZ/0Nn5gPBXn/9AIimgcgRegdilullKGYPlAC4Q1nAxo+udB7ND0aYAzIfa7+qBduRDSI4zTnQztWDkop86SUhwZyo80PX/t6bvL258BwCpkNn1ro90YW9HNUI6XM43h32/TpJCdOnEjPz8/P2b17d25VVVVuc3PzjCNHjuTs27dPAFCO41QLITa2tbUVI9zwqSIie44tFovFEjdszRSdUCgk1q1bN+/3v//9D0pKSv5iyZIlJ1avXr154cKFB6P9DBFJHvreAl1nlDuOs9FxnKO9d+TdJ+Ba3RySm8PZRgQncJ2N9fUhhdc3LYXek35KSrmFh779cjBG4ajvIHGx6epvRdjV3147sdA7atocJyRN4T3pVwJo4HS04mSp73sLI5Xusm5yaNHCYP6eK4TYxa/V2r45+uSjpaVFFBcXZ27fvj33wIEDufX19ZPPnTt3xfbt29ObmpogpTzvOM6u1tbWjxF2kx8govq+PXKLxWLpv9gBusVisfRDhBBfA/ATAPcT0VuXuW0zgF8Q0VO+ry8EsBXAA0T0RtwOto/YsWNH+j/+4z9+bcuWLWvq6upGPvLII6UrV67ckJmZGdXZqJQazu7pOdBFYWFKSkq+EKKpFw89ECJKpeiR6+Ye5noEN3zq+qIJQ3pX8lyl1EIAGUKIUi60k9Kx4otxNIez/t1uLQg/DzXogas/4Bgc3untuStO8tB3T39ttAVhNEFzzaYPtGDBHx/oNd5Md0S7cISbl1OVUgu50dbI6QnbpJTn++QXTDDa2trErl27hhcXF+fs378/t7y8POvw4cOZNTU1o5qbm0ekpqYelVLub2lp2aOU2gdg/6hRo4adPn36C9DOPwngTQB/TcYaDovFYrFY4oWtmTrHyy+/PPu99957aceOHX+5YMGCk6tWrdpy8803l0FfR3WAiITrutfxIH0kgCrHcTZIKav6WtTJCVzD/Olb/N9MBFyro2PketThbByPG0qpK3iQPg3AOXZP7xBJuiaIiFJ8tavpJh9k3NRz9df46teGrrj6g+DVbYtIJ84podc35Uspz/XkfpMJitxN7hcXm+8Jc0e8+VxEuMmVUsNc153HPZMhQogyKWWBlLKyr9//icLp06fT8vLycnbv3p178ODBnMrKyhEnTpzIqa+vHyOlbExJSSlXSu1taWnZA2D/1KlTT1dWVi4E8CSAawDsBbCSiD7s01/EYrFYBgB2gG6xWCz9ECFEEYDJAMZfbhAxkJtBABAKhdLWrl372ObNm5+tqakZ++CDD+5btWrVhpycnLpoP8Pu6UXsngYP0rbGe5DGkXUj/I0G6CbDcOOmFxHpYPb+1YoE3eNOeofiDbzvLxt6j9rmRC20WbCQo5Rqd0QYDZ9072aI3O3mb/jE+xihlJrquu4iAJ67It9xnO39xV0R4FDxYjWD3OT+6EDvPdGlpiOvdlhARDcCcIQQe9i1kpSij65y9uzZlIKCgpySkpLcsrKynMrKyhHV1dU5Z8+eHSOEaEtNTY1o+PC/CiKKKgwRQmQD+AqAmUT05d75TSwWi8Uy0LE1U9d4+eWXZ77//vs/KC4uvmP27NlnVq1alfeJT3yiFJcYpHMU8S0AxgA46jjOxt6IISe9HskTUPoTuLx92ArA2SgJXHG/Vu8uSqkxrusuIaLrAFzg6/uiRLy+p8iVYebz4a0M8zDrV7OO7fK1enfg+n4e1/eDhBD7WKBwPN6P3Vv43OR+cbFZv3rvifaaiXfEN3blPcH1/Uyl1AJoUfcZrpl2JWpPIpYopbB79+7hRUVFufv27cutqKjIOnz4cOaZM2dGXrx4MTMtLe24lHJ/a2vrbtd19yHsJq+Jdp9CPwGfAPB1AH9HRNt66dexWCyWAYsdoFssFks/QwgxE0AJgFeJaG0nbj+g4gijEQqFUr/5zW9+afPmzd86ceLEpPvvv//A6tWrPxo1alRU9TkRDWlra1tIRPOhB2nbHcfZIqXsdoSWTwEe1PDxIutcRHFFAGhK1IbP5eBG2wx2V4wFcIIH6ft66jDozrEYggWzyeAXLDSh44Dca/gkRAS1UmoUiz5mQqcnbHccpyBZ3BVGwyfHbPog2E3uCRbMQXnM1xCQjsy/iSPzMwEc5abQvmR3+iulcPDgwaEFBQW5paWlueXl5dmHDx8ecfr06ZGNjY25qampJx3HKeOGTynCg/JTZIsLi8VisSQBtmbqPq+++uo177777kvbtm27+/rrr6995pln8m6//fY9iD5Ih1Jquuu6t0LHkFfzIP1AT67P2MEcLYHLvx4p6Fr9XDJfsymlsjgd7Sbo6/tt7J6OmqgWL4go3Ve/muJi/8qwCHFxItWvRJTKou5F0LX3URbKH+jtWrQ7UMc0NFNc7HeTRxMXx3oNAZRSVyilFhDR1QCahRDFjuMUSSmjmhaShbq6upT8/PzskpKS3IMHD+ZWVFSMqK6uzj579uwYInLT0tIqiKi0ubl5N3hIDqCciHo9ycJisVgsXccO0C0Wi6WfIYT4F2hF6kwiKu3E7T8AMIEV7ObXnwPwMrQj42RcDjYBCYVCKc8999yDmzdvfu7YsWPT7rvvvrI1a9Z8PG7cuKi7uXmQNp9jyNOFEDt46Hup4Xuaz8FsNnzSjZt6kXX+hk99MhTx3cVwTy8FMAVALbsASmLtQghwqJgNH4dv5iJKw0cIcSGWxxNP2F0xn4jmQrsrSjnevc/dFQHikVwEx997bnJ/06dPBAsU3k25AMAVAOqllEW8RzGhXxtNTU2ysLAwe8eOHbllZWW5lZWVw48ePZpdW1s7RimVkpqaWgFgHzd8vN3kZUTU52srLBaLxWLpCbZm6jmvvfb/s3fn8VHV9/74X+9zZobVrENYBBFccMENRXaspRVr7W1LpVVbt2uxCgoCatVWbbW2WqV6e/G23ltL7c/WWr+tXq/X6wIqGLKRhJCEkG2yhyxksgAJWeac9++P8xk4DDNhyyxJ3s/Hgwd1ck7yOQmFeX8+7+W3577zzjtP79ix49sXXHBBx/3335/xjW98Ix/WQekxbO/vF8Oq/G9SB+khE2UDEloDO3DZK5jt3Ya8ODIeqZVOczxSrGPmMSqpezYAJxHtUkndIePXU/w6Gh+ZER9YTT7GdumgT1iwvb+fB+AsWLFohq7rebFQPR2QPBKYXBxYTX5UcrH6c3FS1eQDxTTNBBWLzgLgUpX+mZqm1cRCAkUopmmivLz8cHKxx+NJrKqqim9qahrX2dk5zul0Num6Xurz+Qp9Pp89ubhBkouFEGJwkwN0IYQYQojIBWAvrAOO+Sd4j3/231xm3qFe02FVZHQw84JwrTeWeb1e/Yknnli+devWn1RVVZ2/bNmy8vXr12+dMmVKc6h7mNllGMZslbE+mojyNU0rAKAH2fAZa7u1E8e2DfQSUVskWtbFOsMwJpmmuYCZLwJwUG1eZNNJzB/kI/Owg1WT2ytU7DPi7Qe0HUMpYUFVV1yu/qwmAqhR1RWl4X7OgPaBR1VHIHg1+eGfw6m0D4wk0zTHq/bulwJgIspXlf4h/96IhJqampGZmZnuwsJCd3l5eVJFRUV8U1OT+8CBA+N1XW91OBzlhmEU9vX17caRDZ9aZh4Um5xCCCHEyZCYaWBt3Ljx7HfeeefnO3bsWD59+vQDq1at2rFs2bJchDhIBwDDMKaqg/TpAFo0TcsEsA9AUpAOXIEJrcE6cMXs+8NIYeYRhmFcqd7fj1WHk6knM2YoRDc0e0KrvxtaH2xzyWElLPjfsw+phAXDMM40TXOeikW7VaV/VgTGt/mryYMlFwerJg9MLm6LlW5ogdS+yWWqvXsygAbVyaswmmvu6urSsrOzE/3JxR6PJ762tjaxtbV1gs/nc7lcrgocSS72x0ylzNwZrTULIYQILzlAF0KIIYSIvgfgTQA/ZOY/Bvn4awBuBzCdmWvVay4AOQDOAPAYgGYAqwB8HcASZk6N0PJjktfr1Z566qlvbd269QmPx3PxN7/5Tc/69eu3Tps2rREADMPAnj17Js6YMWMkEdkPySfh6INZH9RBIAI2fCgG59XFIjV7ej4zXwbAR0Q71OHkQSDkJkMyH1vB3IujN3z8leTeWKgoiCRbdcV8AFNgVVekq0r/U/5eBGy+uW2bb8HaBwa2DoxaNflAYebRagNzNqy/WytVS82ycCUo9Pb2Ul5eXnxOTo67uLjY7fF4EmpqahK9Xu/43t7eMS6Xq5qIint6evKZ2b/hU8LMg751ohBCCHEyJGYKj02bNp355ptv/iwrK+uWyZMnH1q5cuWO5cuX52iaZgBAS0vL6O7ubveECRPOwJEOXBNhHaBptk/VgeAVzEMqoTVc2Jo9falpmgtgJSBUqu5oFf4kA2Z2BlQw25OL7RXM7bb41f5ePWZnxIeLqp6eo6qndZUom65p2r7T+by2avJgycXBqskDk4tjov39qVCx6DmGYcwFcA6ATk3TstWvsCUo1NXVjczIyEhWycXJFRUV8Y2Njf7k4naHw1EWJLm4RpKLhRBi+JEDdCGEGEJUa8G5ACYFy4Ilok0AboO1GVRje30cgF8DuBHWoW8egJ8y82dEtBDWJtE8WNWhdQBeZ+Zn+1nHHQA2BfkQA5jIzFGtxjwVEydOTDznnHPuqq6uvr+hoWHqRRdd1Ovz+ZzV1dV6V1cXdu7cifHjx/sDWy8AL4BWWFUUFwGII6Ji1aawIaoPM8ipeX/XqBaaGqw29z5YB7MudZn9Z2HfeGuhMMzDHgpUdcV8Zr4QVnXFDofDkUVEITPqA6rJj6qOwLEbPv6fRYtKHonpavKBwMyaYRgXqfEOZwJoU9UVeSfTQcGuubnZlZGRkVxQUOAuKytLrqioiG9oaEju6OgYT0TdTqezzDTN3b29vYU4MmevkpmHVXKIEEIIEUo4Yib18WEfNxGRY968eVe1t7c/UVlZuWTMmDGYNm2aXldXpzc2NtKPfvQjPPXUUwDQBVtSMQDDNM1zYVWkd2iatl3X9Z3SjevUmaapmaY5yzbPuwtWe/WROLr9/SEce0AelnnYQwFb49uuVNXTZwAoVwfpFaHimoBE78CYKcF2qf1nYf95xGw1+UAxTdOt2rtfDkAjot0qAfmU9k58Ph/t3LkzMLk4oaWlJaW3tzfO6XRWa5pW3NvbW2Ca5h5YMVMJM7cN6IMJIYQY1OQAXQghREhEdCuAPwP4G6wqjYOwMoMnMfMv+rnvDgB/BHAnrEDELoeZB0XwR0QOAJ8CmAEgxf+6ruvelJSUUVOnTh01YcKEriuvvLL0u9/9bmZSUtI+CjLXTR2iXWaa5iIAiURUpmnaNl3X6yL3NIMLW/MOE+yt7/3tHAHE2S7tg9XWUQPQpGlanqZp5cNhkyFcVHXFXFVdoRFRPhHtIiLYKyJCVJMHtg6UzTdFJSjMVQk1vtbW1vyqqqqiuXPnVgZea5omCgsLz8jOznYXFRW5KyoqEqurqxOam5vHHTp0KNHlcu3VNK24r6+vwDAMf1VEMYAWljf3QgghRMRJ3ET3AVgD65kd6uVDI0eO7Jw5c2ZSXFycOWPGjKbrr78+a/78+WVEdCjY5zFNc5xhGItVomynOkjPGW5dok4GM480TfOoeEm9T0/CkZ+FCasFvhNWomwJEeXput4U6mch+sfMumEYF6sEhQmwYtFMImqAlUQfeFBuT/T2jyI4KrmYiLqi8zSxQyUoXKESFOIB1KSlpZXMmTMnc9SoUcf8fdjS0uIMTC7eu3dvcnt7+3gi6rUlF9urySskuVgIIcSJkAN0IYQQQRHRJFibOH9i5gdO8l7/RtBsZs4Nx/oihYheAdAEoBTW96OcmQ94vV568cUXv7R58+anCwsL5yxZsqR2/fr1X1x66aVVoT6XOki/2DTNxbCC6Apd17fpul4dkYeJQcw8qp8NH/u8Qy8FVJOrTYZu1abwctWmMAGAR7UprBrK1c0DzdY+MJmZ3aZpjofV1t2esGBvH3j4oFxmT5440zTPMAxj9iuvvDLn2Wefdc2fP//Q5MmT83p7ew9VVFTE19fXJ7W1tU1gZnY6nRUAigLm7JUzs4x9EEIIIWKExE0AEX0TwJdgxUz+X/XMbP7jH/9I+tOf/vR4RkbG3WPHjsWKFSvy7r777jSHwxHyAEuNblqoRjcdUiOGsmiIzdc+UcysMXNiQJtvf3LxGNulBxC8mrydiNgwjEmmaS5QCZ2d6vuafaqdkYYjVU1+hi1mcgOYDGA8jiQsAFY1ebDkYkn0PgHMrJmmOaOjo2PuFVdccVZcXBwvWLCgyuFwlDU0NJxRVVUVv2/fvpSurq4klVxc0tfXV2gYRhGOxE37JLlYCCHE6ZADdCGEEEER0VMAngRwtn/230ncOyQ2gk6E1+ull156acEnn3zydH5+/sJrrrmmft26dV9ceeWVFaHuUbO+LjQMYzGsQLtG1/Wt/bV9G8xsbb4Dq8ndAEbZLt2PY9utn/C8Q1u77IWwvq/1uq5v1zStWOYlWjj4nHj/zyNYNXkLrFEEo5n5PFibdI1qs60wWMcFcSyPxzM6MzPTvXv3bnd5eXlSVVVVfHt7+7hzzjknpbGxETU1NQSgDcCHAN4AsAvAXtnwEUIIIWKfxE0nZsuWLXEbN258NC0t7R6Xy+VcsWLFrhUrVqS6XK6Qh+KqM9JCZr4CQK+maRm6rmcS0ZBLJrS9Tw+WXJyII+/Te2GbEQ/Aq2ma/79PKMFAJSjMVwkKPiLKVu2ywzZ3erBRycX2+NUew/qryU3YkosBdDPzZFidGEwiylXf1/boPMXg0tnZqWdkZCTl5eW5S0tL3R6PJ76+vj4pISFhUmJiois7O5tM0zQA7ATwVwCbYSUXSxcFIYQQYSEH6EIIIYIios0ALgPwfQDPA5gJ6yDtnwAeYeYD/dzrn+XXBGAcgA4AnwN4kpl3h3fl0eH1emnjxo2zP/zww2fy8vKuXbBgQcPatWtT586dWw6ravcYzAzTNM83DOMaAJNgHfhu0zStdLAdpAdm4gds+NgPZnthy8TH0Rs+A9JGTX1fzzUMYyGAqeprbNd1PX+4ZPvbqsntVSpuWAfggbPJW2D7majZ5F2BfwbV93W6YRjzAJwL4ICa550zFDcxT1Z3d7eWnZ2dkJub6y4pKXFXVFQk1NbWJni93gk+n2+Ey+WqJKLinp6efGb2zyYvhdXi9RpYbU+/CaAdwBJm3hm9pxFCCCHEiZK46eRkZGSM/fWvf/1wRkbGfQBG3n333QX33XffthEjRoSsgjZNM04d+F4JwCCiLIfDkTEYW14zs5OZk03TPBwv2arJ7e/T2wM6cPl/PzBQsaLqjDSXma8CoKu27mmaprUOyBeIcbYYNljMFDibPHAuuX82+TEJxaZpjjUMYzYzzwYwkoj2aJqWput6fYQeLaZVVlaOysjIcO/evdvt8XiSKisr45uamtwHDhwY73Q6W3RdL/X5fAU+n68IVsxUDKAeVvLCPQBWApgI4BVmvj96TyKEEGKokwN0IYQQQRHRHliHj70AfgkgA8BsAE/Dmse3uJ97lwJYoO7ZD+ASAI/Bypyfx8yF4V19dD377LOzPvjgg2dyc3O/evXVVzetXbs2ddGiRaXo/yD9HFWRfhaARnWQHnOV08zs6mfDxz7Xzd7m277hczCSyQGGYUxWbQovgHXgm64OfAd9+8eAavLDFRG2pAW/Q7C1W7f9PE65faBpmimGYcxj5ktgbWLuHC7VFfX19SPUhk9yWVlZcmVlZXxDQ0Py/v37J+i6vt/hcASbs1fNJzDDlIimAfghgKeZWVppCiGEEIOAxE2nZuvWraNfeeWVtWlpaff7fL6xd911V+GqVau2jRo1KmQ1qTqYnKcOJqEqp9NirXKamYmZ4+3JxbbqZft4pMA23/7YqTWSib9szZ2+yjTNuQBGE1GR6uTVEKk1hJO/mly1Ww88KA9WTX5UcvGpzolnZqdhGJepOelJAGrVn9eSWIvzB1pvby/l5OQk5ObmuouLi90ejyehtrY20ev1ju/r6xulkotLbMnFxQBK+ks48iMiF4CbAOxn5vfD/SxCCCGGLzlAF0IIERQRlcCqMn2UmV+wvb4awEsAvsLMn53E55sKoADAZmZeNtDrjUXPP//8zPfff/8XOTk5X7viiita1qxZk7ZkyZIi9H+QfrY6SJ8GYJ+madt0Xd8dyQBbbfgkBGz4+DcYzrBd2oljD8hjcq6baZpuVAhMIgAAIABJREFUwzAWMPOlAHpV1UrmYKhaCagmD/x52KtUWm1VKi22OfFhe0a1iXm1qloZSURFKklhUFdX+Hw+ys/Pj8vOzvZv+CRWV1cntLS0pPT09MQ5nc5aTdOKe3t7C0zT3IMjGz7DolpHCCGEEEdI3HR68vPzR/7qV79anZaWtrqrqyvx9ttvL1y9evW2sWPHdoa6h5lH+3y+ucx8NazK6Vx14Ls/gksHM480TfOoA3JbcrGuLjNgvU8/KrlYdeGKqdbTzOwwDONy0zTnw0ri8Kjva2Wsd0gLqCY/alwVgHjbpV0InVwclvFUbI1wm6E6eZ0FoFWNI8gbqC5s0dLU1ORKT093FxQUuMvKypIrKiriGxsbkzs6Osbrut7pcDjKDMMo7OvrsycXV51IcrEQQggRbXKALoQQIigiSgMwB8AsZt5le/08WG20HmbmDSf5OT8AcDkzTxrQxca4l1566YJ33333mezs7G/MnDmzdc2aNelLly4tRIiDdOBw5fQ1zHwurAD7C9WCfMCCemYeZdvw8WfjJ8PKjvdv+PgQesNn0LXtVu0f56n2j6Qqp9OiXTkdpAW+fcZesPaBgZs+UU1aUNUVl6vqikQANer7WhrL1RVtbW2O9PT05Pz8fHdZWVmyx+NJ2Lt3b1J7e/sEIupzOp3lqpq8EEc2fCqYedB3MBCRR0RfBnA7gPkAzoQ19z4bVseB4869JaJxAF4A8HUAowHsAvBTZv40bIsWQghxXBI3DYydO3eO2LBhw73bt29f19HRMe62224rWr169daEhISQFamqcvpq9R7Upd7bpw7ke3tm1pk5MURy8WjbpfsRPLm4PZbfDwfDzJphGBeZprkAwAQAe9X3Neod0lQL/CRVTR7487BXk4dKLo5q0oJhGGeapjmPmS8C0K26KGTFWhcFO9M0sWvXrsPJxeXl5Yk1NTUJ+/btG9fd3Z3gcrnqQiQXe6O9djH4SMwkhIglcoAuhBAiKCL6Haz5UqE2gh5i5t+c5OccdhtBdi+//PI577333tNZWVnLLrjggo4HHngg88Ybb9wFK8APyjTNiYZhLFYtyNs1TUtVmeondFiqNnzss7DtGwyjbJd24NgNnxYi2h/tTZJwYOZRhmHMVm0KRxJRoaquaArz13XYfw4BB+XB2gcePihXSQsxXTFvq66YD2AKrOSPdF3Xd0WrusI0TZSUlIzNyspyFxUVuT0eT2JVVVV8c3PzuK6uLrfT6WzQdb2kr6+v0DCMIqgNHwBNLG+UxQAior/Dmm/7dwC7Yf3//iEAVwG4jpk/7+deF4AcWG1ffwxgH4BVAG4EsISZvwjr4oUQQoQkcdPA8nq9rvXr19+dmpr6cEtLy8Rbb711z4MPPrg1OTm5I9Q9zOxS7+3nAxhFRPm6rn+hadoJHeDx0aORApOLEwH4y697ceT9uReA/1C2dSiMiArER0aNLQRwNqzn3a4Su8OWvKt+HnEhYqbAavJQycVhqSYfKKZpJhiGMYeZZ8HqopCv63q6pmn7orWmtrY2R2ZmZnJ+fn5yaWlpckVFRUJ9fX1Se3v7BACG0+ksZ+ainp6eAhyZTe5hGT0lBpDETEKIWCIH6EIIIYIioq8A+BjA48z8nO31tQBeBLCYmbefxOebDivz8yNmvmmg1zuYvPrqq1Pffvvtp7OyspZPmzbt4P3335+1bNmyXPR/kD7eMIxFzHwxrFne29Usb1/ABkPgXPIEHNnw6YFtw8f2e+tgbx13qlTl9BVqsy2eiMo0TUvVNK3mVNsUBrQPDJyxF6yaPHDTJ+Y3fE6Eqq6Yz8wXwqqu2OFwOLKIKGQ7ztPR1dWlZWVlJe3cudNdWlrq9ng88XV1dUmtra3jTdPUnU5nBYCinp4eezV5GTPHdFKCGDqIaBwz7wt4bQyAcgAFzHxdP/euBPDvsObhZqnXdFj/rh5g5nnhW7kQQoj+SNwUHl6v1/nII4/cnpqa+uOGhoazbr755pK1a9d+lpKSErK6XL23v1JVTo8hot3qIL3Z/3G2ZmEfbrcepHqZYSWzenFs7HQw1luZh4t6b79QJXYfUEmyOaeTOBDk52GPYwOryYMlF8dUC/xTobooXGma5hxY49LK1UF6RTj+rJmmifLy8jGZmZn+5OKkysrK+Obm5nGdnZ1up9PZqOt6qc/nK/T5fP7k4mIAjZJcLCJBYiYhRCyRA3QhhBAhEdF7AL4C4FkAGQBmA3gSwMfM/C11zWuw2itNZ+Za9drHAD4DUAjgIIBLATwMYAyABcxcFOFHiUmbNm0688033/xZVlbWLWeeeeahVatW7Vi+fHmOpmlBs/mZ2aUqAObCmp3mg3UIOxKAU13mr172t6vz+tvVAegcrhs+x6PaFM40TXMhrGznWlWRHrIFub+aXLUPDDwoD1ZNfnjzbTBUkw8UVV0xV1VXaKdbXVFTUzMyIyPDXVhY6C4vL0+urKyMb2xsdB88eDBF1/XWEHP26ph50CcliKGJiLYAmKSSTUJd8zGAyardp/31R2H9Gz2ZmRvCu1IhhBChSNwUPl6v1/HYY499/4svvvhxXV3dOcuXLy9dt27dZxMnTmwNdr3qiJRkmubVzHwprFipC9b78rG2S+2zsO2duKI6GinWmabpNgxjgfre9qok2cxQSbK2ZO9gycXBqskDk4vbh0Jy8fEws24YxsVqHMEEAE0qSaHwVP48dnV1aTt27Ei0JRcn1NXVJba2to73+Xwul8vlAbBHVZPbk4vDkuwsxOmSmEkIEQ1ygC6EECIkIhoB4CkAtwKYCGAvgDdgzR7qU9dsAnAbrI2gGvXaBgDXwWrhPApAM4AtAH7BzOXqmoUAHgMwD9amRh2A15n52eOsacjNM9q0adOEt95666msrKzvu93uvu985zt7UlJSfFVVVePr6uqSv/SlL/FNN900ClZGup9/E2gMAB8R7dE0LUPTtKbhsMEQLmrD7TzDMBbASlLYR0Q5RLQPqo2j2vAJ1j4wsHXgkKkmHwiB1RV9fX3lH3zwQcWyZcvSdV0/6tre3l7Kzc1NyM3NdRcXF7s9Hk9CbW1tQktLy/i+vr4xLperioiKe3p68pnZv+FTwsz7o/FsQpwqIooDUA1gMzMv7+e6vQC2MfPNAa/fAOB/ACxl5s1hXawQQoiQJG4KP6/Xqz/xxBPLt27d+pOqqqrzb7jhhsprr722tqGhIammpmZce3v7Gb///e/7ACQB8L+5NAB0wvreOgE0apqWpet68VCoXo4m0zTjVJLsVQCIiHZpmlbCzCNwbHKxPdnbX00emFwsPw8cbps/TXXyOhfAgdTU1D3nnXde2pQpU44ZY1BXVzcyPT3dXVhY6PZ4PEkVFRXxjY2N7gMHDozXdb0tRHJxrSQXi8FEYiYhRLTIAboQQoiII6JbAfwZwN8AvAmr2uIcWNmkv+jnviE1z4iILoFVnXI+gBmapl3EzOcwsw4AI0eOxNSpU30333zzvnvvvbfMVk3uJaJuADBNM15VAMyCdZCe6XA4MmQD4uSo9oFJqprcXx0xCdbMQ81/GdSceNiqI9TcQ/l+nyB/dcV777137T333JMwY8YMY8aMGXt8Pl9rVVVVXENDQ3JHR8cEIupyOp1lpmnu7u3tLcSROXuVzOyL8mMIMSCI6A0AywHMYea8fq7rAfAaM68MeH0ugDQAtzDzW2FdrBBCiIiTuOnwwcmXAcxQv87XNO0S0zTj/NdMnDiRzzrrrO633npr98iRI+0JrR1ExKrb1MWmaS6GlQhboev6Nl3Xq6PzVINTQDW5P2ZKgZU0MsJ26SFYiciBycXDopp8oJimOa6vr2/ewoULr2hqasLixYtbHA5HkdfrHVFTU5PY0tKS0tvbO9blclUTUXFvb2+BaZr25OKQ4w6EGEwkZhJCRIsj2gsQQggROUQ0OtrzfoloEoBXAfyOmR+wfWjrCdz+QwAX4eh5Rp/Dqqb4NayqjMFkDYB/BVADoMQ0zY8AlE6ePLl+wYIFN3366adf6+npIZfLVUJEaQ6H45g55ZqmdWia9oFpmtsMw5jPzPP6+vrmqlZ66aFa6Q1HfPSsePuMvWDV5C1EVENEuQD6TNOcDmvDbgwRFeq6niWH5ifGNE0UFhaekZ2d7Z+zl1hdXZ3Q3NxszJw50yQieu+992bC2hD+EFaFVTYzt0R35UKEFxE9A+AWAPf3txEkhBAismIhZlLrkLjJMgnAOwAOwEqoLDFN8yNd18u+/e1vT6uurv5uQUHBxQsWLKhtaGjYed555+0N/AREZDocjgJmLjRN80LDMBYbhnGnYRg1uq5vDde86cHKNpvcnlwcWE1uwKom9wLYAaBNXXsxgDgi6tU0LV/TtGr53p6YlpYWZ0ZGRnJ+fr67rKzMXVFREbd3795kXdd7L7/8cv2zzz5L6unpWQRr5MN/AXgPQGV3d/cx+wRCDBUSMwkhokkq0IUQYhghoh8AWA9gub8lYBTW8BSseYBn+2f/ncS9Q2qeERElA+hi5qAHsVu2bInbuHHjo+np6fc4nU7nihUrdq1YsWK7y+XqCfU5mXm0z+ebx8xXw5o3naNmeR8I13PEGtuGTzKsTRz/QXngbHJ/+8DDrdf7ax+oqv3nq2p/JqJcNcv7mFZ6w1FHR4cjMzMzadeuXe7S0lJ3RUVFXH19fXJbW9t4Zman0+kBUBQwZ6+cmXuI6GIA6wD8AEAvgK8xc2r0nkaI8LL9W/g4Mz9/AtdLO0IhhIiQWIiZ1DokbgJARDqAFACNHGQT0+v10nPPPXfdli1bfrZnz54rr7/++ur169dvvfDCC+tCfU7VJnuGYRiLYR3Q1+u6vk3TtNLhctjLzNRPcnGc7dJO2Nqt2yrKg1aT26r9F8L6udXpup6qvrfDfhPaNE0UFRWdsWPHjqOSi/ft2zeuq6sr0eVy7dU0rURVk+/BkbipBdYogjsArAVwHoC/MPMPovc0QoSXxExCiGiTA3QhhBhGiOhMALWwNoP+EaU1bAZwGYDvA3gewEwArQD+CeARZg550Dtc3wxv3759zG9+85tH0tPT7wUw6l//9V8LVq5cuW3EiBH9HaSPMgzjatM05wJwEtFOtXExJA57A9oHBs7YC1ZNHrjpc8rtA1WSwtXMPAeAi4gKVJLCvgF4tJhmmiYqKipGZ2Zmunfv3u32eDxJVVVV8U1NTeMOHjw4zul0Nuu6Xurz+Qp8Pl8RVPtAAHuDbXgGIqIJAO4F8AIzS/cEMSSpjaCnADzZX/vdgHs+gnXYcXHA6/6DkDOZuXHAFyuEEMNQLMRMah0SN50Er9dLGzZsuPaTTz75eWFh4ZwlS5bUrl+//otLL720KtQ96iD9HMMwroE1h75RHaQXD5XDXmZ29ZNcHFhNHiy5uPsUvy5M0zzPMIyFAM4C0KJpWqqu64VEZAzEs8WyAwcO6JmZmUl5eXnusrIyt8fjia+vr09qbW2dwMwIklxcAqCMmY/7/VZJJTcC0Jj5nfA+iRDRITGTECIWyAG6EEIME0SkAUgAkAfg78z8EBFpzBzRGWREtAfAVFhVpr8EkAFrDvjTAHKYeXE/9w7reUZbt24dvXHjxrXp6en39/X1nXHXXXcVrly5cuuYMWNCthJn5hGGYcw2TXMegJFElK/r+heaprVGcOmnLEQ1eWD7QHs1+eGD8v6qyQdobS7DMGap720cEZWoTaGQ1S6DRXd3t5adnZ2Qm5vrLikpcXs8noS6urpEr9c73ufzjXC5XBVEVNzd3Z2PIxs+pcx8MMpLFyKmEdETAH4O4Glm/tlJ3HcvgFcAzGXmHeo1HVYr3g5mXhCG5QohxLATKzGTWovETafA6/XSSy+9tOCTTz55Oj8/f+E111xTv27dui+uvPLKilD3qMPes1VF+jQA+zRN+0Id9sb8xqmtmjxYcnGwavJgycVhe07DMKaYprmQmc8HsF/TtHRd13OJqDdcXzNSKisrR2VkZLh3797tLi8vT6qsrPQnF6c4nc59tuRiezV5/YkkFwsxXEnMJISIFXKALoQQwwgROQDkwnrzeBcz+9TrcwDcCWA7M78R5jWUADgXwKPM/ILt9dUAXgLwFWb+LMS9w3YjyC4/P3/kc88998D27dvXdHV1Jd5+++2Fq1ev3jZ27NiQFbvM7DQM4yrTNOfjyBzvL2Khavokqsk7YauIsG36nHI1+UBgZt0wjEtM01wAq+Vhjar2L4v1FpD19fUjMjIy3IWFhe7y8vLkioqKuMbGRvf+/fvH67re4XA4ygzD2N3X17cbRzZ8aph5yFeNiPAhoi8DuB3AfABnwpqZmQ1rgyT3OPfeAWBTkA8xgInM3DzAyx0wRLQewAsA/g/W4cdRmDlTXfcarO/PdH/LXiJyAcgBcAaAxwA0A1gF4OsAlsjIAyGEGDixEDOprydx02nwer20cePG2R9++OEzeXl51y5YsKBh7dq1qXPnzi2H9b4hKHXYu5iZzwXQqg7S86MZb/jZq8lVvGSPmxzqMns1eWBy8SlVkw8U0zRTDMNYwMyXAOjWNC1L1/UsIuqK5rqOp7u7W8vJyTkqubi2tjbB6/VOUMnFVQD29Pb2FjCzP2Yq7a9LhBDHIzGTxExCiOiTA3QhhBgmiEhnZoOI3gEwjpkXqhncjwBYDeAQgP8E8Gw4Az0iSgMwB8AsZt5le/08WFWsDzPzhhD3DrtWhP3ZuXPniA0bNty7ffv2dR0dHeN+8IMf7FmzZs3nCQkJIX9+zOwwDOMKNZMujoiK1EF62NtY2arJ3QACD8rt1eReIvLCVh2haVpLtDd8joeZSc1SXAgrwG1WFem7o7nh5vP5aNeuXXHZ2dnu4uJit8fjSaypqYlvaWkZ39PTE+d0Oms0TSsOmLNXwsxt0VqzGNqI6O8AxgH4O4DdsBJPHgJwFYDrmPnzfu69A8AfYR1glAR8OCeWkzuI6DMAIasFmVlX120CcBuszaAa2/3jAPwaVsvO0bCqI38a6vBECCHEyYuVmEmtReKmAfLss8/O+uCDD57Jzc396pw5cxoffPDBtEWLFpWgn4N00zQnGoaxmJkvANCu3tfnhbv9+ElUkx9E6OTimN7sNU0z3jCMecx8JQAmolxd19OjPW6soaFhREZGRnJBQYG7rKwsubKyMr6xsTG5o6NjvK7rB1VycWFAcnF1LL//FIOXxEzBScwkhIgkOUAXQohhhoh+C+ArAD4A8C0AEwD8FcATzNwUga//OwD3IPRG0EPM/JsQ98o8oyC8Xq9z/fr1P0xNTX2opaVl0i233LJn7dq1W5OTk0NuQKiq6cvUQXoiEZVqmrZN1/X601mL2vA5w7/R49/0gRXsBWsfGLjpE9Vq8oGgWkBOVQfp58LacEtTG2594fq6ra2tzvT09OSCgoLksrKyZI/HE793797k9vb2CUTU53Q6y0zT3N3b21uIIxs+FcwctjUJEQwRjWPmfQGvjQFQDqCAma/r517/ZtDs41VeCCGEEKcq2jGTWoPETQPs+eefn/n+++//Iicn52uzZs3a9+CDD6Zfe+21Rej/IH28YRiL1PfygKZp21X78dN6D22rJvfPJA9VTR4qubjndL5+LGDm0YZhXG2a5tUARhBRga7r28PZJc3n81F+fn5cTk5OcnFxsbu8vDyxuro6oaWlJaWnpyfe6XTWBiQXl8BKLvaGa01CBCMxkxBCRJ8coAshxBBHROSfr0VEZwFYCauCohfAZgBPAchTlRZhn+9HRF8B8DGAx5n5OdvrawG8CGAxM28Pca/MM+qH1+t1PvLII7enpqb+uKGh4aybb765ZO3atZ+lpKS0h7qHmTXVfnwRrM0aj67r23Rdrwl1j7rPXk1uPyi3V5Pb2wcePiiPhfaBkWKa5gTVpvBiAIc0TctUbQpP6flN00RJScnYrKwsd1FRkdvj8SRWVVUlNDc3j+vq6kp2uVx7NU0r6evrKzQMwz5nr5nlTZ+IcUS0BcAkZr6wn2tkM0gIIcSAi7WYSa1D4qYweemlly549913n8nOzv7GzJkzW9esWZO+dOnSQvR/kO5WB+mXAOhSCbLZ1M8c74Bq8sDk4jNslx5E6OTiIf8enpldhmHMMk1zHqwuaaWq4r/2VD9nW1ubIzMzM3nXrl3u0tLS5IqKivi9e/cmt7W1TSAin0ouLgpILvYw86Cfyy6GNomZhBAicuQAXQghhgEi0gDcAuAZWMH6WAAPMvNv1ccp1OGaf4OIiGbAmr30+uluGBHRe7AqOp4FkAFgNoAnAXzMzN9S18g8o1Pk9Xodjz/++C3btm17rK6ubvpNN91Utm7dus8nTZoUMmuemckwjItM01wMIAVAtaZp21S1Q7J90wehq8kDN32GxYbPiTBNM1G1KbwCgHno0KHc2tra/JkzZzYEu76zs1PPzMxMzMvLc5eWlro9Hk98XV1dUltb2wTTNDWn01kBoKinp6cARzZ8ypj5UAQfS4gBQ0RxAKoBbGbm5f1c55/n1wSrpWEHgM8BPMnMuyOwVCGEEENUrMVM6vNK3BRGv/3tb8999913n87Kyvr2BRdc0PHAAw9k3njjjbtgjZUKSr2vX8jMlwPo0TQtQ9O0nQDGqmryw+OqELyaPFhy8aCvJh8IqkvaTNUlzQ2gpqioKO+SSy7Zqev6MdebpomysrIxmZmZ7qKiInd5eXlSdXV1fHNz87jOzk630+ls1HW9tK+vr8AwjCIciZuaJLlYDEYSMwkhRGTJAboQQgxxRPRVWPP6boD1hnkTrEzU7zLzu/1tBAV8ng0A1gK4k5n/fJprGgGriuNWABMB7AXwBoCn/e2kZZ7R6fN6vfqTTz5509atW39SWVk5Y9myZeXr16/fOmXKlGb7dYZhuIgo2TRNf0X5NACTcGSzBzi6mvzwQflwqiYfCMw8xufzzXn99dfnPfHEE46lS5e2Xnrppdvb2tq4vLw8qaqqKr6xsdF98ODBFIfD4dV1vczn8xX4fD77hk99JKqehIgkInoDwHIAc5g5r5/rlgJYAOsQYT+AS2AdDCQCmMfMhRFYrhBCiCEmFmMm9fkkboqAV199derbb7/9dFZW1vLp06cfWLVq1Y5ly5blwnaQbhgGEVG8v4rcNM1JAM7G0YnFwJFq8sDk4g5JLj4xzEymaZ5/8ODBhVdfffXk5ORk39KlSwvGjBlTVlFRkeTxeOJra2uTWltbJ5im6XA6nR4Ae4IkF3dF90mEGFgSMwkhRGTJAboQQgxRRDQf1kbKjwBUAvgJgA9htSGsAvBHZn70OJ/DX0lxPoB/ADgA4Bpm7jvRTaRIIqJrAATbEGJYQUJWP/f6M3SD3TuRmZuDfGxQ8Hq92pNPPvntzZs3/7yqqmrGrFmz2iZNmqTX1dWNqq6uHnnffffRPffc47/8IAD/jD0w81RY2f8Nuq5v0zStRDZ+Tk5vby/l5OQk7Ny5011cXOz2eDwJLS0t7gkTJpy1Z88eR2trKzRNazBN80MAn8La8Cll5v1RXroQEUFEzwB4HMD9zPy7U7h/KoACWJUYywZ6fUIIIYau4RgzARI3hbJp06YzX3/99Wezs7O/Gx8f75s7d273/v37R9TU1Izu7u52ZGRk+C/1QSUXq/fsbliH6SYR7XA4HOlE1Bmlxxi0ampqRmZmZroLCwvd5eXlSR6PJz4+Pn6qz+eL37VrF2BV/GebpvkOrPd+JQBqJblYDAcSMwkhROQ5jn+JEEKIwYaICMBSAN8D8EsALzNzi/qYE0AbrDZO/bYitAWitwE4C8CjaiNIZ2YjzI9xOh6FVTlidyIZtgzgTliBuF3I1uexjIhWArgGwAwA5wMYBQCZmZnus88+myZOnOi77rrrms4999xiXdfLNU1rCWwfyMwwTXOaYRiLDcP4nmEYzZqmbdN1vUgO0o/W3NzsSk9PTy4oKHCXlZUlV1RUxDc0NCTv379/gqZpnQ6Ho8w0zd29vb0fAyjxeDzFCQkJewHcaprmIwDugrXx9l05PBfDBRE9BWsj6PFT2QgCAGauJqJUAHMHdHFCCCGGNImZAEjcBCK6CMADsGKmC2BV+qOzsxObN28+4+yzz+YZM2Z0nnPOOXUAchwOR9BqcmYe7fP55jHz1X19fXOIKEfX9e2aph2I+EPFsN7eXsrLy4vPyck5nFxcU1OT2NLSMr6vr2+My+WqIqLinp6eVGb2V5OX6Lp+tmEYj5imeTOsn9WDzPxxdJ9GiMiQmEkIIaJDDtCFEGIIUps7TxHRBvtBHBE51GZOKaygE6E2gmyVFNMBfAuAB8Cf1IdjPcO7vL+qiePYzcy5A7qa6LkC1jzzDFg/uxIAxfPmzauZP3/+V7Zs2fKzt99++8r9+/dPnjJlSsXFF198zOw9IoKu65W6rlcahnGWYRiLTdO8yTRNr6ZpX+i6XkBEsf7nYcCYpomCgoK4HTt2uPfs2eP2eDyJ1dXVCS0tLeMOHTqU4HK56jRNK+nr68s2DGMP1IaPYRgt/XzaP6jWm98C8B0ArZF4FiGiTW0EPQVrFt/z0V6PEEKI4UViJgASNwHWnPj5sGKlP6jfiwGU/uEPfxj5pz/96SdpaWl3FhUVjZ80adLEO++8sypYIjERdTmdzi3MnGYYxhzTNOf4fL6riGinruupmqZ1RPaxoqu5udmVkZFxOLm4srIybu/evckdHR0TiOiQ0+ksV8nFW3Ck7Xpld3e3L8Sn3AXg+0T0UwDrAfQXXwkxZEjMJIQQ0SMt3IUQYogLVi2hDuvmA7iKmYNmxPvvU2/W18Gal/fv/VVSRLtFoa0V4U3M/M+TvPcOWHMOZw+RjaDj8nq99OKLL35p8+bNTxcWFs5ZsmRJ7fr167+49NJLq/q7zzCMM03TXMTMMwC0aZqWquv6LiKK9QqbE9bR0eHIyMhI2rVrl7usrCzZ4/HE7927N7mtrW08M5sul8vDzEUBc/Y8zHxMEoJ1LyWrAAAgAElEQVQQ4lhE9ASAn8Oa4fqz0/xc02Ftqn7EzDcNwPKEEEIMM8MpZlJrkLjpJPzf//1f4u9+97vHMzIy7h4zZgytWLEi7+67705zOBx9oe5h5hGGYcw2TXMegJFElK/r+heapg2ZZFnTNFFYWHhGTk6Ou6io6HBycXNz87hDhw4lulyuvZqmFff19RXYkouL/Z0ehBD9k5hJCCGiSw7QhRBiGCKiHwH4HYCUYMGrrZJiKoD/Vi8vYObO4234qFaIWjTaFdo2gppgzaHrApAO4Blm3n6ce/2z/JpgtWrsgNXO8Elm3h3GZUed1+ulf/u3f1vw0Ucf/Tw/P3/R4sWL69evX//FlVdeWdHffaZpTjAMYxEzXwRgvzpI30lEoaoGYoppmqioqBidmZnp3r17t7u8vDypqqoqvqmpaVxnZ+c4p9PZpOt6ic/nK/T5fEVQ1eQAGqK96SkGPyK6DMCzAC6B9XfOIVh/vl5h5r+cwP3jALwA4OsARsPaDPkpM38atkUPECJaD2vt/wfg6cCPM3Omuu41ALcDmM7Mteq1j2H9PV8I4CCASwE8DGAMrH+niiLxDEIIIYa+oRozqa8vcdMp2LJlS9zGjRsfTU9Pv8fpdDpXrFixa8WKFdtdLlfIJFpmdhqGcZVpmvMBjCGiQnWQvi+CSz8tBw4c0DMzM5Py8vLcpaWlbo/HE19fX5/U1tY2gZnZ6XR6APiTi/0V/OXM3B3dlYuhYLjGTRIzCSFE9MkBuhBCDENENAbAQwA2B9sgsVVS/BTWm+yfMfNLoSopiGgSgFmwKnD3+D8HrH9nIta6kIguhxU4fA5r/t65av0zANzAzJ/0c+9SAAtgtTvfDys4ewxAIoB5zHwiswAHNa/XSxs3bpz94YcfPpOXl3ft/PnzG9atW5c6d+7cclhzDoMyTXOcOkifCaBT07Q0XdeziShkRUYkdXd3a1lZWYk7d+48vOFTW1ub2NraOsHn87lcLpcHwJ6enp5CHKkmL2XmzuiuXAxlauP6ewBSAdTD2sz4PoBbYG3o/LKfe10AcgDEAfgxgH0AVgG4EcASZv4ivKs/PUT0GYDFoT7OzLq6bhOsebLTmblGvbYBwHUApgAYBaAZwBYAv2Dm8jAvXQghxDAyVGMm9XUlbjoNGRkZY1944YWH09PT7wUw6u677y647777to0YMaK/g3SHYRhXmKa5EEAcERWpg/TGyK28fx6PJ1hysfvgwYMpTqezWdf1Mp/PV2BLLi4GsFeSi0U4Dde4SWImIYSIPjlAF0IIcRRbJcVkAO8BcAJYxMztIVobumDNbf4egMsAVAN4jJnTQ3z+0czcFd6nOOrrxQMoANDCzLNO8t6p6t7NzLwsHOuLVc8+++ysDz744JmdO3d+Zfbs2c1r167dvmjRohL0f5CeZBjGQma+DEC3pmnpuq7vIKKItDWvq6sbmZGRkVxYWOguLy9PrqysjGtoaHAfOHBgvK7r7Q6Ho9wwjMK+vr7dOLLhUxPpDUsh+kNE6QAmMvPZ/VyzEsC/w9qkzlKv6bCqKQ4w87xIrFUIIYQYroZazKS+psRNJ2nr1q2jX3nllbVpaWn3+3y+sXfddVfhqlWrto0aNepQqHuYWTcM4zLTNBcBSCCiUk3Ttum6Xh+JNXd3d2vZ2dkJubm5/uTihNra2gSv1zvB5/ONcLlclURU3N3dnY+jk4sPRmJ9QpwoiZuEEEKEmxygCyGEOIptM+hhWJUEv2TmF4NVUtiudQKYDOAsAD+CNSvw35l5Q8D1Z8Gal7cLwMOROrgkov9Q6xp9sjOqiegDAJcz86SwLC7GPf/88zPff//9Z3Jycm644oorWtasWZO2ZMmSIvR/kJ5gGMYCZr4CQJ+maZm6rmcQ0Wm38PP5fLRz58743Nzc5OLi4sNz9lpaWlJ6e3vjnE5ntaZpxb29vQWmafrn7JUwc/vpfm0hIoGI/gfARcx8Tj/XfAxgshqfYH/9UVjtDSczc0N4VyqEEEIMX0MxZlJfW+KmU5Cfnz/yueeee2D79u1rurq6Em+//fbC1atXbxs7dmzIjlbMrBmGcYk6SE8G4NF1fZuu6zUDsaaGhoYRGRkZyQUFBe6ysrLkysrK+IaGhuT9+/dP0HV9v8PhKAuRXByVsQJCnCyJm4QQQoSbHKALIYQ4BhGNB5AJK5C/IbCtlX9jiIhGAbgYwIWw5kJvVh//GawKi6WwzYsmomUA/g6rzdZzEXwe/0bQKGbuPcl7h+1GkN1LL710wX//938/vWPHjn+ZOXNm2+rVq9Ovv/76AvR/kH6GYRjzmfkqACYRZTkcjnQiOm41TWtrqzM9PT05Pz/fXVZWllxRURG/d+/e5Pb29vFE1Ot0OstM09zd29vrb7teAqCCmWOibbwQJ8o/AxVW29PvAngZwGpm/n0/9+wFsI2Zbw54/QYA/wNgqf/vYyGEEEKEx1CLmdTXlrjpNOzcuXPEhg0b7k1LS1vb3t6ecttttxWtXr16a0JCwoFQ9zAzGYZxkWmaiwGkAKjWdX2rpmmV1tvE0Hw+H+Xn58dlZ2e7i4uL3eXl5Qk1NTWJLS0tKT09PXFOp7MmILm4BFZyceuAPrgQESBxkxBCiEiTA3QhhBBBEdH9sDJyDwB4A8CbsFq3HbJd8zsAdwFoBOBSL/9UXd8F4KvM/Jm6diKAVwFcBGAhMzeGmg84wM+RCCAfQJM6yD2Ze6fDqvz4iJlvCsf6BpuXX375nPfee+/pHTt2fPv888/fv3r16swbb7xxF4CQlTHMPMbn881j5tkPPfSQg5kbV65c+d6MGTOaiouLx2ZlZbmLiorcHo8nsaqqKmHfvn3jurq6klwu115N00r7+voKDcOwz9nbJ3P2xFCh/h79kfpPH6xKs387zj09AF5j5pUBr88FkAbgFmZ+KxzrFUIIIcQRQyVmUl9b4qYB4vV6XevXr787NTX1oZaWlkm33nrrngcffHBrcnJyR6h7mJlM05xhGMbi/Pz8iY8++mjvHXfcsf3WW2/dtn//fkdgcnF9fX1Se3v7RCLqU8nFRbbk4mJYycUnlQQhRCyTuEkIIUSkyQG6EEKIfhHRYwAegRWgvA7gb8ycTUSXwNok+RWAdwD0APg2gHthZQR3wgpGPlGfZxmAvwJYDeCfzNxi+xraQLQmJKI3AFQByAHQCuB8AOsATAdwvW1j6jUAtwOYzsy16rWPAXwGoBDAQQCXAngYwBgAC5i56HTXN5S8+uqrU99+++2ns7Kylk+bNu3AqlWrdnznO9/JRZCD9M7OTj0jIyOptLR0UnZ29jWffvpp4qFDh2AYhg+Az+VylQPY09PTU4AjGz7l9o1HIYYqNTs1Rf36Bqy/Qx9j5l/3c49sBAkhhBAxZDDFTOpzSdwUAV6v1/nII4/cnpqa+uOGhoazbr755pK1a9d+lpKScsx4KdM0UVJSMjYrK8tdXV191ZYtWy7YvXu3PmLECF9PT4/mcDgaHA5HSZDk4mZJLhbDgcRNQgghIk0O0IUQQpwQInoIwOMAqgEsAvAArHl/X2LmXCIiZmYiSoJVuVAMYCUzl6pA5xkAdwB4F8BoAG4ALzLz3wZwjY8A+B6AaQDGwtoM+gLAr5g513bdJgC3wdoIqlGvbQBwHYApAEYBaAawBcAvmLmciK6BtVEUiAHMY+as46xtHIAXAHwd1vPvgtWW8dNTf+Lo27Rp05lvvvnmU1lZWbdOmjTp0KJFi0o0TfN6PJ6kysrK+KamJvfBgwdTHA6HV9f1Up/PVzB+/PiG7u7ui71e73Wwfk6vw/oZVUT5cYSIOtU69W4AZ9o3zQOukVaEQgghRAwaDDGTWqfETRHk9XodP/nJT27dunXro3V1ddOvu+66qqlTp+6pq6s7w+PxxNfV1SW1traON01TdzqdFQCKenp6Ci677LLxu3btmg1gNqyEhV8A+H8yp1wIiZuEEEKEnxygCyGEOClE9CVm/pyIvgrgHwDWAPizP4gnogcB/AbWJscv1Wv/AuDPALwAXgPgAXA1gHsA/IyZN0T+SU6ObSPoUQCfB3y4kJlDzvUmIhes6o44AD8GsA/AKgA3AlgSOC9xMPrrX/86/o9//OPPP/vss7ucTufHvb29eczsn01ewszHzP0jorGwssYfgpUV/pMIL1uImENEd8L6ezLkBjMRfQRgMjNfHPD6o7DayJ7JzI3hXqsQQgghghuuMRMgcVN/vF6v/uSTT970l7/85TednZ2HiGhzX1+fvZq8LliXASJaCKvt/2wAZweLrYQYbiRuEkIIEW5ygC6EEOKUEJEbwB8AzINVQdwMYCGAr8AK/m9n5iIimgTgJQCLAdxqawcYB+AvAC6EFfDsC/I1BqxN4emybQTdxMz/PMl7VwL4d9gCOyLSYVVTHGDmeQO93sGEiEYBcMhGkBAAEf0ZwC0AJjCzN8Q19wJ4BcBcZt6hXvP/ndLBzAsitV4hhBBChDbcYiZA4qZwIqIUZm6O9jqEiAUSNwkhhAg3LdoLEEIIMTgxcwszfwtWZcF8ACMAXACgBcA7ttl3s2C13/svWG0BQUQ6M+8H8AasOXtjQnwNU13vCOOjRMK3YFVhH86KVtUnbwC4mogmRm1lMYCZD8nh+fBDRJcR0ftEVE1EXUTkJaI0Ivr+Cdx7BxGZQX4ZRJQSifWfLiJ6lYheIKLlRLSYiJYR0d8AfB9Wq1avuu41Iuojoim22/8IoAjA20R0CxEtAfA2gPNgVWsJIYQQIgZIzHTSJG7qhxyeD08SN0ncJIQQIjqGwptrIYQQUcTMmwBsAgAimg5r1t/H6r8nAfgmABeAVACBs9rmAOgAcBaAKtXSe4F6fTKAXGb+PTP7IvAoJ+oVInoLQBeAdADPMPP249wzE8C2IK/nq98vBtAwcEsUYlBIAFAD4K8A6mFtCn8fwP9HRFP97Uz7wQDuhDUmwC5o9UEMSgNwF4DbYX0vDsKqhPgBM79pu04DQOoXAICZe4noywB+DeC3sOaD5gG4nplTI7N8IYQQQpyoYRgzARI3CTFQJG6SuEkIIUQUSAt3IYQQA4qIzmfmUvW/vwXg32Bt7HyVmT9VlRQGEY2D2jQCcC0ztxPRPwDcAKANwA4Al8FK9lrJzO8FfJ2pAJ4A8Fdm/jQCz3U5rIDtc1iB5rkAHgYwA8ANzPxJP/f2wJrxvTLg9bmwgsFbmPmtMC1diEGFiNIBTGTms/u55g5Y1QSzmTk3UmsTQgghhBgIQzVmUl9T4iYhIkDiJiGEECK8pIW7EEKIAWXbCHICuBzAFAD/DeCbRBSnWvABVhvDGQD+pjaCHodVeXE/M0+CtelyLYDtANYSUVLAl5oL4F9hbTSFHTPnMfM6Zn6Pmbcz8+uwKj8aADwfiTUIMUy04NjKKyGEEEKIIWOoxkyAxE1CRJDETUIIIUQYSQt3IYQQYcHMfQB+RkT/CWAqgD8AGEtEJbA2UL4B4C0A/0VEZwFYDiuxa7q6vwNABxG9DGtDaB6A/wUAIjoTVguvPFhtzKKCmTuI6H0APyKiEczcE+JSL4DAzSyo1xiDp3WaEAOOiAjW//cTAXwXwFIAq0/kVgD/qyqzOmBVOT3JzLvDtFQhhBBCiAE1HGImQOImIQaCxE1CCCFEZEkFuhBCiLBi5r3MnA5gJayZdg8CmAbgWViVE62wqiouA/AKgFuJ6CAR+ed4VQJoBXCB7dPOB/BVABtiaNZffzNRCgBcEuT1S9XvhQO/HCEGjf8A0AegGcDLAB5m5t8f555GAL8AcDeALwH4KYDZADKIaGb4liqEEEIIMfCGUcwESNwkxKmSuEkIIYSIIJmBLoQQIqKIaAqARlVt4X/tNgAvAFgEoBpWcLcOwJkAMmAFet9g5v9V9/8ewNkArmDm3og+gA0RJQLIB9DEzFf1c929sDa65jLzDvWaDmAXgA5mXhCJ9QoRi4hoMoAU9esbAO4F8Bgz//okP89UWJuum5l52YAvVAghhBAiQoZSzARI3CTEQJC4SQghhIgsOUAXQggRdUR0IYCdAB5l5pfVa3EAlsHaFBoF4HxmZiJaBquN4T3MvCmCa3wDQBWAHFjVHeertU0HcD0zf6auew3WLMLpzFyrXnOp+84A8BisjPFVAL4OYAkzp0bqOYSIdUT0H7A2hM9k5paTvPcDAJermaBCCCGEEEPGYIiZ1JokbhIiAiRuEkIIIcJLWrgLIYSIOmbeA6ut2K1E9A0iGg3gADP/CdYcv+vVRtBEAHcAqADwZoSXmQ/gawBeA/AJgGdgtRCc598EUjRYM8bI/4Kq+PgygM8A/BbAewDGw3ou2QQS4mhZABxQsz2FEEIIIcSgiZkAiZuEiBSJm4QQQogwkgp0IYQQMYGIUmC1JPwegM0APgDgBfARM7era/4FwP+DNQfwP6O11kghoh8C+E8AB5k57jjX3gEgWHUJA5jIzM1hWKIQA46I/gzgFgATmNl7EvdNh9Xe8yNmvilc64t1RHQtgF8CeIaZP4j2eoQQQggxcCRmOpbETGK4krjp9EjcJIQQ4nikAl0IIURMYOZmZr4DwCwALQBuBnAjgHOAw5tFtwOoBfBGtNYZKUQ0CdbmWP1J3Mawqk3m2n7Ng7WpJmIcEf2QiEwi2n+C148joj8R0T4i6iSiNCL6crjXOVCI6FUieoGIlhPRYiJaRkR/A/B9AC/6N4GI6DUi6lOzPP33fkxEj6nqq2uJaA2AbQB8AJ6MygNFGRH539ePBjADwCVRXI4QQgghwkBipqNJzDT8DLeYCZC4aaBJ3CSEEOJEOaK9ACGEEMKOmYsA3ElESQDG+OfhAbgawFIAP2bmrqgtMHJeBfA5gHYA3zmJ+3Yzc25YViTCJmDzL/4ErncB+BRAHIAHAOyDNR/yQyJawsxfhHG5AyUNwF2wNnkTAByEVQnxA2a2txs9pr0ngAIAtwKYAmveZzOsKqxfMHN5+JcePUREOPI9MfhIOyn/71Ww/hydH/nVCSGEECISJGY6TGKmYWSYxkyAxE2nROImIYQQp0tauAshhIh5RPT/t3f/Mb+fdX3Hn+8WKcJKwfJD0A7F6TSldD+CwoaKDhPQhRkhMyJzJYMtwuaPMBIaN5YwzObcxEDYbBYxWzRz4g+cQVfHAtE6iWZmoz+crI61tS1YWpjtmNL2XPvj8z1w9/b0x/Gcc5+75348kjv3fa7v5/O9r/s+35Oc9/v1va7rwrYz9F5WPWWtdc9ZntIZNTOvqt5RXVp9f/XyR7gd4buq52kGPfrMzC+0rQL4ZI/s7/t1ba+RF6y1fmM3dn5bI+XutdYLzvCUOQ1m5inV66o71lr/amYes9a6b2ZmneR/0o/fMzNPrX6qOm+t9bVnYt4AwOGjZlIznevUTEeXugmAs8EW7gAcemutu9veMf7SI9AIenr1tupNa63bTvb26r0zc9/M3DkzPzMzl57+WXI67Zp/L2xbDfFIfXP1O8cbQVVrrfvbtur8ypl5xumdJafTbjVE1ZPbGkFX7H38eBNoz/aCe++9ZGbeNDPvn5lrZ+Zfz8xz9zSO7qpuq555xn4AAODQUTM98ttTMz3qqJmOJnUTAGeTAB2AR4W11sfWWu8/2/M4AO+sblhrXXWS9320emv1t6oXVf+gel71wZl5zmmdIafNKTT/nlN96ATjx8c0AQ+J2Zy/+5j6bKOn7d/tf60u3o3fNzNfMDMvnJnL1lrH9j3X09peL3+9uqH6d9VXVb81M1+7W01xf1sj6KKZ+fwD+SEBgENBzfSw1EyPQmqmo0HdBMBh4wx0ADgkZuYV1TdVl5/svWutq6ur9wxdMzO/2Hbm2Vuqbzktk+R0+5M2/y5ue8f8fne1raq5+FQnxme399s3dnH1T6qr11o/83DbBu4eu/9BHrt7Zm6rvm63heBV1Te0bU1518z8SPVDu+ZObQ3fF1evrd67e95/Xv3Kbk7fXn2k7Sy/x1R/pq3ZBABwTlAzHUlqpkNO3QTAucgKdAA4BGbmCW3ns72j+tjMXDQzT6oeu3v8opl5/Mk851rrpuqa6vmne76cuj3Nv9ee7bnwQDPz0pm54kEaPJ+uXtO2aqmHagLNzIUz8/yZ+e6Z+acz89qZuWDfZbdWx6pf3X399dU3tq2U+IHqlbvnenLb6+WH11rvXmt9aq31R2utT1cfqL6ybQVVbSsp7q2+/OR/egCAw0nNdPSomQ43dRMA5zIBOgAcDk+pnl69ofrE7uOu6tuqP7X784+ftdlxWp2G5t+d1eedYPzzqrV7nAexa9Bcuvt69j32JW2rFP7CCe6b3fmid1ZPmZnHPcT3+EvVv6l+om2b0EurN7atdLpsz6W3tq2cuLP6wbXWb661fr36G9VvVK/brbL409UF1eUz85bdWX63zMwfVt9b3dRn/29/S3V39RUn83sBADjk1ExHiJrp7FM3AXCU2cIdAA6Hj7Z7Z/Y+V1ZfU72kkyzwZ+bZ1Vf3wG0KORz2Nv/+/gke/0T1nh58G8lrq8tOMP7c3efrTnWC57gfqb5tZl6z1npX1cyctzs77x9Wt7dt47nfeW3b/91cfXH15N21n7Fna8JntDVy3rzWum732DOrn6u+Z2b+zlrrvrYGzvnVdWutm2fm/Kq11idn5uerN7dtKXhH2wqJb6weV/3P6qer3949xx1tzZ/amksfr77slH5LAACHi5rpaFEznX3qJgCOLAE6ABwCa60/ajuP6wFm5tXV/WutX90z9qPVd1TPXmvdshv75er9bU2Ae9qaAm9se4f2m8/4D8DJOtXm389V75yZ5621frNq10B4VfXBtZbz2x7CWuvbZ+b+6l/OzP+rfnKtdWxmvqxtu793rrU+foJbj6+6uL76uuqp1e17z/PbszXhe9Za98/Ms2bmiratAr+ibbvAe9uaO/+jrZF0V1tz8Pj3OLbn+1zQ1jz879XvVk9aa730AZPaVoN8/u66/9X22rmt+pI/ye8HAOAwUjMdOWqms0zdBMBRJkAHgMNv/1lh57UVi3u3ULu27cyvS6rPrX6/el/11rXWjQcxSR65U23+Ve+qXl+9e2aubPv7fn31pdVfOcPTf1Tb07R5ffX46u1tjZd/33a24nnVu/dde9zxr/9b9a1tqyU+1PZv8QH/TndNoG+u/tlu6Her/7L7/Irq4t34x9pWPjxnd999e57micevWWt9ambeVv3CzPx422vg5uoJbc2ll1T/qbpqrfV/Z+ae6ukz8/i11qdO+hcFAPDoomY6x6iZzi51EwBHnTPQAeAQW2u9eq110QnGHrPWunnP2BvWWpettZ601rpgrXXJWuuKw94ImpnXzMyxmfmDR3Dt39xdu//j/pl52kHM94A8bPNvrfXp6uvbVtC8vfoPbe/Ef8la65oDmuej0lpr7bYdvLv6e23bBf7QzHxf9Q3VT6+1rp+Zx+xrAtVn/24+VH1O9cx945+x2w703dUN1V+t/tpa6/uqn60urI6/Zu9q20rwi2bmlcfPFpyZL6i+s61JdMNu7r9Sva5thcS/qK6pfr160+66/7xnCt9dfaEmEABwrlMzPeBaNdPxC9RMp0TdBMBRZwU6AHBW7M41+8G2Avaih7n8uFVdUf3OvvGTOuvwsFprvbp69cON7cbvONE4D2+37eCstW6fme9sa6r84+ojbc21/SsaPnPf7svf3n2+ZDf+xxpB1Ze3NfKuXGt9eM/4i9oae8+YmfPXWvfNzM1t24i+vfrzM3NL9eK21THftda6pz6zsuMnZuaXqr9Yfbr68FrrAecJ7ub0iUf+GwEA4DBSM/1xaqaDo24C4CgToAMAZ8tV1QeqT1YvP4n7rl9r/dYZmRFHxp6z926emTdUX1XdtVtFsX8Lwv333joz91bP2jVz7j/BZf+n7TzNF8/M7W3n931Tdfwcvi9q2wrx7t3HE9tWdjyt+rvV/66+p/qpvXPeze2utm0HAQA4t6mZOKvUTQAcVQJ0AODAzcyrqhdWl1bff5anwxG3awZdW33pw519t9vG8Fj1e20rKS6sPjkzj23bnvDCatZavzYz/7H6gbYtDu+tntX2ev/e3dfn7572J9u2FPy1tdYfVv/oIeb6oA0qAADOHWomDht1EwBHiQAdADhQM/P06m3Vm9Zat+2OLnvEt1fvnZmntr1T/QPVm9da15/2iXIk7Fk18XvV86ovrD78EKspzquOVddWl1cvm5n7q8vazti7vLpxZr6r+tbqb7edvfjx6t9Wv1T97O5MxuPf/8bqUJ+9CQDAwVEzcdiomwA4agToAMBBe2d1w1rrqpO876PVW6sPVn/QVnhfWX1wZl6w1rru9E6TI2Lazon8SHVBWzPnw3vGH8xHqpdVP9a2SuLWtjP+fr56X3XrblXE23cfJ2RlBAAAJ6Bm4rBRNwFwpAjQAYADMzOvaDvP7PKTvXetdXV19Z6ha2bmF9ve0f6W6ltOyyQ5qm5sO3vvz7atdniwZT7Hdp9/uK3pc/1a644zPz0AAI4CNROHnLoJgCNBgA4AHIiZeUL1jt3Hx2bmorZi+7G7xy+q7n2oc9T2W2vdNDPXVM8/A1PmaLm9elz1xbs/HzvRRbtz/Fpr3VTddDBTAwDgKFAz8SigbgLgSBi7nwAAB2FmntW2fdvqwd+l/p611kmtititqPhza61nnuIUOcJm5nOqJ1kVAQDA2aJm4rBTNwFwVFiBDgAclI9WLzrB+JXV11Qvqe48mSecmWdXX90DtymEk7bWurfSBAIA4GxSM3GoqZsAOCqsQAcAzqqZ+bHq5WutJ+4Z+9HqO6pnr7Vu2Y39cvX+6rrqnuq51RurJ1R/ea11w0HPHQAA4ExTMwEAHCwr0AGAw2D/O/rOa9uycO+2hddWr6wuqT63+v3qfdVb11o3HsQkAQAAzhI1EwDAAbECHQAAAAAAAADa3qkIAAAAAAAAAEeeAB0AAAAAAAAAEuk57k4AAAckSURBVKADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAACoBOgAAAAAAAABUAnQAAAAAAAAAqAToAAAAAAAAAFAJ0AEAAAAAAACgEqADAAAAAAAAQCVABwAAAAAAAIBKgA4AAAAAAAAAlQAdAAAAAAAAAKr6/20JUN1IJ2OGAAAAAElFTkSuQmCC","arguments":{},"addedWidgets":{},"removedWidgets":[]},"errorSummary":"Command skipped","error":null,"startTime":1.465536417344E12,"submitTime":1.46553640403E12,"finishTime":1.465536418345E12,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"3511caf8-aa50-475b-a287-3e45fac81559"},{"version":"CommandV1","origId":124241,"guid":"f29776da-0eea-425a-b973-12a19b97910d","subtype":"command","commandType":"auto","position":6.0,"command":"%md\nThe figures showed above shows a good performance of the high order spectral clustering algorithm.","commandVersion":0,"state":"finished","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":1.465536404034E12,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"a user","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"5025c389-1dad-4b3f-977e-bc0f0ee32a5a"},{"version":"CommandV1","origId":135761,"guid":"cc431dc6-abf8-40be-92bd-40c8f04f2e4e","subtype":"command","commandType":"auto","position":7.0,"command":"%md\n\n# [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)\n\n\n## Course Project - High Order Spectral Clustering\n### by Xin Zhao\n\n*supported by* [](https://databricks.com/)\nand \n[](https://www.awseducate.com/microsite/CommunitiesEngageHome)","commandVersion":0,"state":"error","results":null,"errorSummary":null,"error":null,"startTime":0.0,"submitTime":0.0,"finishTime":0.0,"collapsed":false,"bindings":{},"inputWidgets":{},"displayType":"table","width":"auto","height":"auto","xColumns":null,"yColumns":null,"pivotColumns":null,"pivotAggregation":null,"customPlotOptions":{},"commentThread":[],"commentsVisible":false,"parentHierarchy":[],"diffInserts":[],"diffDeletes":[],"globalVars":{},"latestUser":"","commandTitle":"","showCommandTitle":false,"hideCommandCode":false,"hideCommandResult":false,"iPythonMetadata":null,"nuid":"30952775-bddc-4519-b3a6-95d265f5f8b7"}],"dashboards":[],"guid":"9945fd64-cc83-4b87-ae6b-079c3238d9fd","globalVars":{},"iPythonMetadata":null,"inputWidgets":{}};</script> <script src="https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js" onerror="window.mainJsLoadError = true;"></script> </head> <body> <script> if (window.mainJsLoadError) { var u = 'https://databricks-prod-cloudfront.cloud.databricks.com/static/201602081754420800-0c2673ac858e227cad536fdb45d140aeded238db/js/notebook-main.js'; var b = document.getElementsByTagName('body')[0]; var c = document.createElement('div'); c.innerHTML = ('<h1>Network Error</h1>' + '<p><b>Please check your network connection and try again.</b></p>' + '<p>Could not load a required resource: ' + u + '</p>'); c.style.margin = '30px'; c.style.padding = '20px 50px'; c.style.backgroundColor = '#f5f5f5'; c.style.borderRadius = '5px'; b.appendChild(c); } </script> </body> </html>