From 4a959883e7567e204896e03386990ec5aafe558c Mon Sep 17 00:00:00 2001 From: Tao Luo Date: Mon, 12 Aug 2019 09:42:00 +0800 Subject: [PATCH] remove unused aws_benchmarking and go directory (#19103) test=develop --- go/glide.lock | 233 ------ go/glide.yaml | 33 - paddle/scripts/paddle_build.sh | 3 - tools/aws_benchmarking/README.md | 184 ----- tools/aws_benchmarking/client/Dockerfile | 7 - .../client/cluster_launcher.py | 415 ---------- .../aws_benchmarking/client/requirements.txt | 6 - tools/aws_benchmarking/diagram.png | Bin 40790 -> 0 bytes tools/aws_benchmarking/server/Dockerfile | 7 - .../aws_benchmarking/server/cluster_master.py | 735 ------------------ tools/aws_benchmarking/server/logs/master.log | 0 .../server/pserver.sh.template | 2 - .../aws_benchmarking/server/requirements.txt | 4 - .../server/trainer.sh.template | 2 - 14 files changed, 1631 deletions(-) delete mode 100644 go/glide.lock delete mode 100644 go/glide.yaml delete mode 100644 tools/aws_benchmarking/README.md delete mode 100644 tools/aws_benchmarking/client/Dockerfile delete mode 100644 tools/aws_benchmarking/client/cluster_launcher.py delete mode 100644 tools/aws_benchmarking/client/requirements.txt delete mode 100644 tools/aws_benchmarking/diagram.png delete mode 100644 tools/aws_benchmarking/server/Dockerfile delete mode 100644 tools/aws_benchmarking/server/cluster_master.py delete mode 100644 tools/aws_benchmarking/server/logs/master.log delete mode 100644 tools/aws_benchmarking/server/pserver.sh.template delete mode 100644 tools/aws_benchmarking/server/requirements.txt delete mode 100644 tools/aws_benchmarking/server/trainer.sh.template diff --git a/go/glide.lock b/go/glide.lock deleted file mode 100644 index d15fc934db..0000000000 --- a/go/glide.lock +++ /dev/null @@ -1,233 +0,0 @@ -hash: 107c058cf5c9163a75d40eef2273a793c36112683c25d72aa8288827fdde3a19 -updated: 2017-10-30T03:46:19.137696069Z -imports: -- name: github.com/alecthomas/gometalinter - version: bae2f1293d092fd8167939d5108d1b025eaef9de -- name: github.com/beorn7/perks - version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 - subpackages: - - quantile -- name: github.com/boltdb/bolt - version: 583e8937c61f1af6513608ccc75c97b6abdf4ff9 -- name: github.com/cockroachdb/cmux - version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92 -- name: github.com/coreos/etcd - version: f1d7dd87da3e8feab4aaf675b8e29c6a5ed5f58b - subpackages: - - alarm - - auth - - auth/authpb - - client - - clientv3 - - clientv3/concurrency - - compactor - - discovery - - embed - - error - - etcdserver - - etcdserver/api - - etcdserver/api/etcdhttp - - etcdserver/api/v2http - - etcdserver/api/v2http/httptypes - - etcdserver/api/v3client - - etcdserver/api/v3election - - etcdserver/api/v3election/v3electionpb - - etcdserver/api/v3election/v3electionpb/gw - - etcdserver/api/v3lock - - etcdserver/api/v3lock/v3lockpb - - etcdserver/api/v3lock/v3lockpb/gw - - etcdserver/api/v3rpc - - etcdserver/api/v3rpc/rpctypes - - etcdserver/auth - - etcdserver/etcdserverpb - - etcdserver/etcdserverpb/gw - - etcdserver/membership - - etcdserver/stats - - lease - - lease/leasehttp - - lease/leasepb - - mvcc - - mvcc/backend - - mvcc/mvccpb - - pkg/adt - - pkg/contention - - pkg/cors - - pkg/cpuutil - - pkg/crc - - pkg/debugutil - - pkg/fileutil - - pkg/httputil - - pkg/idutil - - pkg/ioutil - - pkg/logutil - - pkg/monotime - - pkg/netutil - - pkg/pathutil - - pkg/pbutil - - pkg/runtime - - pkg/schedule - - pkg/srv - - pkg/tlsutil - - pkg/transport - - pkg/types - - pkg/wait - - proxy/grpcproxy/adapter - - raft - - raft/raftpb - - rafthttp - - snap - - snap/snappb - - store - - version - - wal - - wal/walpb -- name: github.com/coreos/go-semver - version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6 - subpackages: - - semver -- name: github.com/coreos/go-systemd - version: 48702e0da86bd25e76cfef347e2adeb434a0d0a6 - subpackages: - - daemon - - journal - - util -- name: github.com/coreos/pkg - version: 3ac0863d7acf3bc44daf49afef8919af12f704ef - subpackages: - - capnslog -- name: github.com/dgrijalva/jwt-go - version: d2709f9f1f31ebcda9651b03077758c1f3a0018c -- name: github.com/ghodss/yaml - version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7 -- name: github.com/go-stack/stack - version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf -- name: github.com/gogo/protobuf - version: 909568be09de550ed094403c2bf8a261b5bb730a - subpackages: - - proto -- name: github.com/golang/protobuf - version: 4bd1920723d7b7c925de087aa32e2187708897f7 - subpackages: - - jsonpb - - proto -- name: github.com/golang/snappy - version: 553a641470496b2327abcac10b36396bd98e45c9 -- name: github.com/google/btree - version: 925471ac9e2131377a91e1595defec898166fe49 -- name: github.com/grpc-ecosystem/go-grpc-prometheus - version: 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 -- name: github.com/grpc-ecosystem/grpc-gateway - version: 18d159699f2e83fc5bb9ef2f79465ca3f3122676 - subpackages: - - runtime - - runtime/internal - - utilities -- name: github.com/inconshreveable/log15 - version: 0decfc6c20d9ca0ad143b0e89dcaa20f810b4fb3 -- name: github.com/jonboulle/clockwork - version: 2eee05ed794112d45db504eb05aa693efd2b8b09 -- name: github.com/mattn/go-colorable - version: 5411d3eea5978e6cdc258b30de592b60df6aba96 -- name: github.com/mattn/go-isatty - version: 57fdcb988a5c543893cc61bce354a6e24ab70022 -- name: github.com/matttproud/golang_protobuf_extensions - version: c12348ce28de40eed0136aa2b644d0ee0650e56c - subpackages: - - pbutil -- name: github.com/namsral/flag - version: 71ceffbeb0ba60fccc853971bb3ed4d7d90bfd04 -- name: github.com/PaddlePaddle/recordio - version: 0432dee9fd4b24fb6840fb20a8c055b0c933fb81 -- name: github.com/prometheus/client_golang - version: c5b7fccd204277076155f10851dad72b76a49317 - subpackages: - - prometheus -- name: github.com/prometheus/client_model - version: 6f3806018612930941127f2a7c6c453ba2c527d2 - subpackages: - - go -- name: github.com/prometheus/common - version: 49fee292b27bfff7f354ee0f64e1bc4850462edf - subpackages: - - expfmt - - internal/bitbucket.org/ww/goautoneg - - model -- name: github.com/prometheus/procfs - version: a1dba9ce8baed984a2495b658c82687f8157b98f - subpackages: - - xfs -- name: github.com/satori/go.uuid - version: 879c5887cd475cd7864858769793b2ceb0d44feb -- name: github.com/sirupsen/logrus - version: f006c2ac4710855cf0f916dd6b77acf6b048dc6e -- name: github.com/topicai/candy - version: 1b9030d056fa9f8c4b1f9c91b52fe4b8ab4cd8cc -- name: github.com/ugorji/go - version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74 - subpackages: - - codec -- name: github.com/xiang90/probing - version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2 -- name: golang.org/x/crypto - version: 9419663f5a44be8b34ca85f08abc5fe1be11f8a3 - repo: https://github.com/golang/crypto.git - vcs: git - subpackages: - - bcrypt - - blowfish - - ssh/terminal -- name: golang.org/x/net - version: c8c74377599bd978aee1cf3b9b63a8634051cec2 - subpackages: - - context - - http2 - - http2/hpack - - idna - - internal/timeseries - - lex/httplex - - trace -- name: golang.org/x/sys - version: e48874b42435b4347fc52bdee0424a52abc974d7 - repo: https://github.com/golang/sys.git - vcs: git - subpackages: - - unix - - windows -- name: golang.org/x/text - version: 836efe42bb4aa16aaa17b9c155d8813d336ed720 - repo: https://github.com/golang/text.git - vcs: git - subpackages: - - secure/bidirule - - transform - - unicode/bidi - - unicode/norm -- name: google.golang.org/grpc - version: 8050b9cbc271307e5a716a9d782803d09b0d6f2d - subpackages: - - codes - - credentials - - grpclog - - internal - - keepalive - - metadata - - naming - - peer - - stats - - tap - - transport -- name: gopkg.in/yaml.v2 - version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b -testImports: -- name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib -- name: github.com/stretchr/testify - version: 05e8a0eda380579888eb53c394909df027f06991 - subpackages: - - assert diff --git a/go/glide.yaml b/go/glide.yaml deleted file mode 100644 index c5d66694ac..0000000000 --- a/go/glide.yaml +++ /dev/null @@ -1,33 +0,0 @@ -package: github.com/PaddlePaddle/Paddle/go -import: -- package: github.com/PaddlePaddle/recordio -- package: github.com/coreos/etcd - version: ^3.2.1 - subpackages: - - clientv3 - - clientv3/concurrency - - embed - - etcdserver -- package: github.com/namsral/flag - version: ^1.7.4-pre -- package: github.com/sirupsen/logrus - version: ^1.0.0 -- package: github.com/topicai/candy -- package: golang.org/x/crypto - repo: https://github.com/golang/crypto.git - vcs: git -- package: golang.org/x/sys - repo: https://github.com/golang/sys.git - vcs: git -- package: golang.org/x/text - repo: https://github.com/golang/text.git - vcs: git -- package: github.com/satori/go.uuid - version: v1.1.0 -- package: github.com/alecthomas/gometalinter - version: v1.2.1 -- package: github.com/inconshreveable/log15 - version: v2.13 -- package: github.com/go-stack/stack - version: v1.6.0 -- package: github.com/golang/protobuf diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index e65cf31243..7f6793a249 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -265,9 +265,6 @@ function check_style() { # set up go environment for running gometalinter mkdir -p $GOPATH/src/github.com/PaddlePaddle/ ln -sf ${PADDLE_ROOT} $GOPATH/src/github.com/PaddlePaddle/Paddle - mkdir -p ./build/go - cp go/glide.* build/go - cd build/go; glide install; cd - export PATH=/usr/bin:$PATH pre-commit install diff --git a/tools/aws_benchmarking/README.md b/tools/aws_benchmarking/README.md deleted file mode 100644 index 4fdd4b0de4..0000000000 --- a/tools/aws_benchmarking/README.md +++ /dev/null @@ -1,184 +0,0 @@ -# AWS benchmark testing tool -This is an automation tool for deploying paddlepaddle benchmark testing to AWS. - -## Features - - - subnet creation to fit just the amount of ec2 instances required. - - pserver and trainer ec2 instances allocation, and instance state verification - - nvidia-docker ready for GPU training - - Instances and network element garbage collection when a task is accomplished or an error occurred - - Test log is collected in realtime - - Web service for checking log or tearing down the testing setup - - No testing code change needed - - Lots of optional configuration options - - ## Usages - - ### Prerequisites - - - You have a working AWS account - - You have [AWS Command Line Interface](https://aws.amazon.com/cli/) installed - - Your AWS cli is bind with a account which has `AmazonEC2FullAccess` permission, and it's set as default credential. - - You have key pair created and pem file downloaded. - - You have a default VPC in the region you want to run the test. - - You have a Security Group created for the VPC mentioned above, which allows port 22 and the port you want to expose your control web service (5436 by default) - - If your test is supposed to run in a GPU machine, especially a multi card GPU machine (p2, p3 series), you might need to contact amazon to raise the limit which allows no more than 1 GPU instance at a time. - - ### Start a benchmark test - -#### Create training image - -*What to expect in this step:* - -*You will have your training logic packed with paddle runtime in a docker image, and be able to be picked up by AWS instance for training.* - -Training python script and PaddlePaddle runtime are supposed to be packed into one docker image. Use PaddlePaddle production images as base image and create the training images with the docker file as follows: - -```Dockerfile -FROM paddlepaddle/paddle:latest-gpu - -ENV HOME /root -COPY ./ /root/ -WORKDIR /root -RUN pip install -r /root/requirements.txt -ENTRYPOINT ["python", "my_training.py"] -``` - -***Please Note*** -Training nodes will run your `ENTRYPOINT` script with the following environment variables: - - - `TASK_NAME`: unique name to identify this training process. - - `TRAINING_ROLE`: current node's role in this training process, either "PSERVER" or "TRAINER" - - `PSERVER_HOSTS`: comma separated value of pserver end points, I.E. "192.168.1.2:5436,192.168.1.3:5436" - - `PSERVERS`: same as above - - `TRAINERS`: trainer count - - `SERVER_ENDPOINT`: current server end point if the node role is a pserver - - `TRAINER_INDEX`: an integer to identify the index of current trainer if the node role is a trainer. - - `PADDLE_INIT_TRAINER_ID`: same as above - - Now we have a working distributed training script which takes advantage of node environment variables and docker file to generate the training image. Run the following command: - - ```bash - docker build -t myreponname/paddle_benchmark . - ``` - - Now you have the image built and tagged with `myreponame/paddle_benchmark`, let's push it to dockerhub so that it can be picked up by out AWS instance. - - ```bash - docker push myreponame/paddle_benchmark - ``` - -#### Create instances and start training - -*What to expect in this step* - -*you will be asked to provide some basic settings to config your training, and this tool will have your training started and monitored* - -Now let's start the training process: - -```bash -docker run -i -v $HOME/.aws:/root/.aws -v :/root/.pem \ -putcn/paddle_aws_client \ ---action create \ ---key_name \ ---security_group_id \ ---docker_image myreponame/paddle_benchmark \ ---pserver_count 2 \ ---trainer_count 2 \ ---trainer_command batch_size:20,local:no,device:CPU -``` - -Now just wait until you see this: -``` -master server finished init process, visit http://XXX:XXX/status to check master log -``` -That means you can turn off your laptop and your cluster is creating instances, starting training process, collecting logs and eventually shut all pservers and trainers down when training is finished. - -#### Post creation operations - -To access the master log: - -```bash -docker run -i -v $HOME/.aws:/root/.aws \ -putcn/paddle_aws_client \ ---action status \ ---master_server_public_ip \ ---master_server_port -``` - -To tear down the training setup: - -```bash -docker run -i -v $HOME/.aws:/root/.aws \ -putcn/paddle_aws_client \ ---action cleanup \ ---master_server_public_ip \ ---master_server_port -``` - -To retrieve training logs -TBD - -### Tech details - -*What to expect in this step* - -*You will understand what is happening behind the scene, and how to check the training log, how to tear down the training on the fly, etc.* - -Let's understand what is happening under the hood when you run above command in your laptop - -![alt](diagram.png) - -There are 4 roles in the figure above: - - client: your laptop - - master: who tasks to aws api server to create/tear down instances, and monitor training process - - AWS api server: the one who actually creates and manages instances - - pservers and trainers: training instances - -When you run the `docker run` command above, what it actually does is to ask aws api service to create a subnet (step 1) and a master instance (step 2), and pass all the parameters the client collected or generated (step 3). The master is kept as minimum hardware config to keep the running cost low. - -Then when the master is up and running, it will ask the aws api server to create the heavy lifting training instances who are expensive to run (step 4). And the master will start training process as soon as they are done initializing (step 5). - -Meanwhile, the master will expose a web service for client to check training log or even tear the training setup down by a web service call. - -if you are creating the training with client docker container, and also monitoring your aws dashboard, you will initially see a instance tagged with `ROLE=MASTER` and `TASK_NAME=_master` starts, then you will see several instances tagged with `ROLE=PSERVER` and `ROLE=TRAINER` starts. -When the training is finished, pservers and trainers will be terminated. All their logs are kept in master node's docker env. - -Master exposes 4 major services: - - - GET `/status`: return master log - - GET `/logs`: return list of log file names - - GET `/log/`: return a particular log by log file name - - POST `/cleanup`: teardown the whole setup - - -### Parameters - - - key_name: required, aws key pair name - - security_group_id: required, the security group id associated with your VPC - - vpc_id: The VPC in which you wish to run test, if not provided, this tool will use your default VPC. - - subnet_id: The Subnet_id in which you wish to run test, if not provided, this tool will create a new sub net to run test. - - pserver_instance_type: your pserver instance type, c5.2xlarge by default, which is a memory optimized machine. - - trainer_instance_type: your trainer instance type, p2.8xlarge by default, which is a GPU machine with 8 cards. - - task_name: the name you want to identify your job, if not provided, this tool will generate one for you. - - pserver_image_id: ami id for system image. Please note, although the default one has nvidia-docker installed, pserver is always launched with `docker` instead of `nvidia-docker`, please DO NOT init your training program with GPU place. - - pserver_command: pserver start command, format example: python,vgg.py,batch_size:128,is_local:no, which will be translated as `python vgg.py --batch_size 128 --is_local no` when trying to start the training in pserver. "--device CPU" is passed as default. - - trainer_image_id: ami id for system image, default one has nvidia-docker ready. - - trainer_command: trainer start command. Format is the same as pserver's, "--device GPU" is passed as default. - - availability_zone: aws zone id to place ec2 instances, us-east-2a by default. - - trainer_count: Trainer count, 1 by default. - - pserver_count: Pserver count, 1 by default. - - action: create|cleanup|status, "create" by default. - - pserver_port: the port for pserver to open service, 5436 by default. - - docker_image: the training docker image id. - - master_service_port: the port for master to open service, 5436 by default. - - master_server_public_ip: the master service ip, this is required when action is not "create" - - master_docker_image: master's docker image id, "putcn/paddle_aws_master:latest" by default - - no_clean_up: no instance termination when training is finished or failed when this value is set "yes". This is for debug purpose, so that you can inspect into the instances when the process is finished. - - -### Trouble shooting - - 1. How to check logs - - Master log is served at `http://:/status`, and you can list all the log files from `http://:/logs`, and access either one of them by `http://:/log/` diff --git a/tools/aws_benchmarking/client/Dockerfile b/tools/aws_benchmarking/client/Dockerfile deleted file mode 100644 index 812c5d4bce..0000000000 --- a/tools/aws_benchmarking/client/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:2.7.14-stretch - -ENV HOME /root -COPY ./ /root/ -WORKDIR /root -RUN pip install -r /root/requirements.txt -ENTRYPOINT ["python", "cluster_launcher.py"] \ No newline at end of file diff --git a/tools/aws_benchmarking/client/cluster_launcher.py b/tools/aws_benchmarking/client/cluster_launcher.py deleted file mode 100644 index 12333202b9..0000000000 --- a/tools/aws_benchmarking/client/cluster_launcher.py +++ /dev/null @@ -1,415 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import time -import math -import logging -import copy - -import netaddr -import boto3 -import namesgenerator -import paramiko -from scp import SCPClient -import requests - - -def str2bool(v): - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument( - '--key_name', type=str, default="", help="required, key pair name") -parser.add_argument( - '--security_group_id', - type=str, - default="", - help="required, the security group id associated with your VPC") - -parser.add_argument( - '--vpc_id', - type=str, - default="", - help="The VPC in which you wish to run test") -parser.add_argument( - '--subnet_id', - type=str, - default="", - help="The Subnet_id in which you wish to run test") - -parser.add_argument( - '--pserver_instance_type', - type=str, - default="c5.2xlarge", - help="your pserver instance type, c5.2xlarge by default") -parser.add_argument( - '--trainer_instance_type', - type=str, - default="p2.8xlarge", - help="your trainer instance type, p2.8xlarge by default") - -parser.add_argument( - '--task_name', - type=str, - default="", - help="the name you want to identify your job") -parser.add_argument( - '--pserver_image_id', - type=str, - default="ami-da2c1cbf", - help="ami id for system image, default one has nvidia-docker ready, \ - use ami-1ae93962 for us-east-2") - -parser.add_argument( - '--pserver_command', - type=str, - default="", - help="pserver start command, format example: python,vgg.py,batch_size:128,is_local:yes" -) - -parser.add_argument( - '--trainer_image_id', - type=str, - default="ami-da2c1cbf", - help="ami id for system image, default one has nvidia-docker ready, \ - use ami-1ae93962 for us-west-2") - -parser.add_argument( - '--trainer_command', - type=str, - default="", - help="trainer start command, format example: python,vgg.py,batch_size:128,is_local:yes" -) - -parser.add_argument( - '--availability_zone', - type=str, - default="us-east-2a", - help="aws zone id to place ec2 instances") - -parser.add_argument( - '--trainer_count', type=int, default=1, help="Trainer count") - -parser.add_argument( - '--pserver_count', type=int, default=1, help="Pserver count") - -parser.add_argument( - '--action', type=str, default="create", help="create|cleanup|status") - -parser.add_argument('--pem_path', type=str, help="private key file") - -parser.add_argument( - '--pserver_port', type=str, default="5436", help="pserver port") - -parser.add_argument( - '--docker_image', type=str, default="busybox", help="training docker image") - -parser.add_argument( - '--master_server_port', type=int, default=5436, help="master server port") - -parser.add_argument( - '--master_server_public_ip', type=str, help="master server public ip") - -parser.add_argument( - '--master_docker_image', - type=str, - default="putcn/paddle_aws_master:latest", - help="master docker image id") - -parser.add_argument( - '--no_clean_up', - type=str2bool, - default=False, - help="whether to clean up after training") - -args = parser.parse_args() - -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - -ec2client = boto3.client('ec2') - - -def print_arguments(): - print('----------- Configuration Arguments -----------') - for arg, value in sorted(vars(args).iteritems()): - print('%s: %s' % (arg, value)) - print('------------------------------------------------') - - -def create_subnet(): - # if no vpc id provided, list vpcs - logging.info("start creating subnet") - if not args.vpc_id: - logging.info("no vpc provided, trying to find the default one") - vpcs_desc = ec2client.describe_vpcs( - Filters=[{ - "Name": "isDefault", - "Values": ["true", ] - }], ) - if len(vpcs_desc["Vpcs"]) == 0: - raise ValueError('No default VPC') - args.vpc_id = vpcs_desc["Vpcs"][0]["VpcId"] - vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] - - logging.info("default vpc fount with id %s and CidrBlock %s" % - (args.vpc_id, vpc_cidrBlock)) - - if not vpc_cidrBlock: - logging.info("trying to find cidrblock for vpc") - vpcs_desc = ec2client.describe_vpcs( - Filters=[{ - "Name": "vpc-id", - "Values": [args.vpc_id, ], - }], ) - if len(vpcs_desc["Vpcs"]) == 0: - raise ValueError('No VPC found') - vpc_cidrBlock = vpcs_desc["Vpcs"][0]["CidrBlock"] - logging.info("cidrblock for vpc is %s" % vpc_cidrBlock) - - # list subnets in vpc in order to create a new one - - logging.info("trying to find ip blocks for new subnet") - subnets_desc = ec2client.describe_subnets( - Filters=[{ - "Name": "vpc-id", - "Values": [args.vpc_id, ], - }], ) - - ips_taken = [] - for subnet_dec in subnets_desc["Subnets"]: - ips_taken.append(subnet_dec["CidrBlock"]) - - ip_blocks_avaliable = netaddr.IPSet( - [vpc_cidrBlock]) ^ netaddr.IPSet(ips_taken) - # adding 10 addresses as buffer - cidr_prefix = 32 - math.ceil( - math.log(args.pserver_count + args.trainer_count + 10, 2)) - if cidr_prefix <= 16: - raise ValueError('Too many nodes to fit in current VPC') - - for ipnetwork in ip_blocks_avaliable.iter_cidrs(): - try: - subnet_cidr = ipnetwork.subnet(int(cidr_prefix)).next() - logging.info("subnet ip block found %s" % (subnet_cidr)) - break - except Exception: - pass - - if not subnet_cidr: - raise ValueError( - 'No avaliable subnet to fit required nodes in current VPC') - - logging.info("trying to create subnet") - subnet_desc = ec2client.create_subnet( - CidrBlock=str(subnet_cidr), - VpcId=args.vpc_id, - AvailabilityZone=args.availability_zone) - - subnet_id = subnet_desc["Subnet"]["SubnetId"] - - subnet_waiter = ec2client.get_waiter('subnet_available') - # sleep for 1s before checking its state - time.sleep(1) - subnet_waiter.wait(SubnetIds=[subnet_id, ]) - - logging.info("subnet created") - - logging.info("adding tags to newly created subnet") - ec2client.create_tags( - Resources=[subnet_id, ], - Tags=[{ - "Key": "Task_name", - 'Value': args.task_name - }]) - return subnet_id - - -def run_instances(image_id, instance_type, count=1, role="MASTER", cmd=""): - response = ec2client.run_instances( - ImageId=image_id, - InstanceType=instance_type, - MaxCount=count, - MinCount=count, - UserData=cmd, - DryRun=False, - InstanceInitiatedShutdownBehavior="stop", - KeyName=args.key_name, - Placement={'AvailabilityZone': args.availability_zone}, - NetworkInterfaces=[{ - 'DeviceIndex': 0, - 'SubnetId': args.subnet_id, - "AssociatePublicIpAddress": True, - 'Groups': args.security_group_ids - }], - TagSpecifications=[{ - 'ResourceType': "instance", - 'Tags': [{ - "Key": 'Task_name', - "Value": args.task_name + "_master" - }, { - "Key": 'Role', - "Value": role - }] - }]) - - instance_ids = [] - for instance in response["Instances"]: - instance_ids.append(instance["InstanceId"]) - - if len(instance_ids) > 0: - logging.info(str(len(instance_ids)) + " instance(s) created") - else: - logging.info("no instance created") - #create waiter to make sure it's running - - logging.info("waiting for instance to become accessible") - waiter = ec2client.get_waiter('instance_status_ok') - waiter.wait( - Filters=[{ - "Name": "instance-status.status", - "Values": ["ok"] - }, { - "Name": "instance-status.reachability", - "Values": ["passed"] - }, { - "Name": "instance-state-name", - "Values": ["running"] - }], - InstanceIds=instance_ids) - - instances_response = ec2client.describe_instances(InstanceIds=instance_ids) - - return instances_response["Reservations"][0]["Instances"] - - -def generate_task_name(): - return namesgenerator.get_random_name() - - -def init_args(): - - if not args.task_name: - args.task_name = generate_task_name() - logging.info("task name generated %s" % (args.task_name)) - - if not args.pem_path: - args.pem_path = os.path.expanduser("~") + "/" + args.key_name + ".pem" - if args.security_group_id: - args.security_group_ids = (args.security_group_id, ) - - -def create(): - - init_args() - - # create subnet - if not args.subnet_id: - args.subnet_id = create_subnet() - - # create master node - - master_instance_response = run_instances( - image_id="ami-7a05351f", instance_type="t2.nano") - - logging.info("master server started") - - args.master_server_public_ip = master_instance_response[0][ - "PublicIpAddress"] - args.master_server_ip = master_instance_response[0]["PrivateIpAddress"] - - logging.info("master server started, master_ip=%s, task_name=%s" % - (args.master_server_public_ip, args.task_name)) - - # cp config file and pems to master node - - ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect( - hostname=args.master_server_public_ip, username="ubuntu", pkey=ssh_key) - - with SCPClient(ssh_client.get_transport()) as scp: - scp.put(os.path.expanduser("~") + "/" + ".aws", - recursive=True, - remote_path='/home/ubuntu/') - scp.put(args.pem_path, - remote_path='/home/ubuntu/' + args.key_name + ".pem") - - logging.info("credentials and pem copied to master") - - # set arguments and start docker - kick_off_cmd = "docker run -d -v /home/ubuntu/.aws:/root/.aws/" - kick_off_cmd += " -v /home/ubuntu/" + args.key_name + ".pem:/root/" + args.key_name + ".pem" - kick_off_cmd += " -v /home/ubuntu/logs/:/root/logs/" - kick_off_cmd += " -p " + str(args.master_server_port) + ":" + str( - args.master_server_port) - kick_off_cmd += " " + args.master_docker_image - - args_to_pass = copy.copy(args) - args_to_pass.action = "serve" - del args_to_pass.pem_path - del args_to_pass.security_group_ids - del args_to_pass.master_docker_image - del args_to_pass.master_server_public_ip - for arg, value in sorted(vars(args_to_pass).iteritems()): - if value: - kick_off_cmd += ' --%s %s' % (arg, value) - - logging.info(kick_off_cmd) - stdin, stdout, stderr = ssh_client.exec_command(command=kick_off_cmd) - return_code = stdout.channel.recv_exit_status() - logging.info(return_code) - if return_code != 0: - raise Exception("Error while kicking off master") - - logging.info( - "master server finished init process, visit %s to check master log" % - (get_master_web_url("/status"))) - - -def cleanup(): - print requests.post(get_master_web_url("/cleanup")).text - - -def status(): - print requests.post(get_master_web_url("/status")).text - - -def get_master_web_url(path): - return "http://" + args.master_server_public_ip + ":" + str( - args.master_server_port) + path - - -if __name__ == "__main__": - print_arguments() - if args.action == "create": - if not args.key_name or not args.security_group_id: - raise ValueError("key_name and security_group_id are required") - create() - elif args.action == "cleanup": - if not args.master_server_public_ip: - raise ValueError("master_server_public_ip is required") - cleanup() - elif args.action == "status": - if not args.master_server_public_ip: - raise ValueError("master_server_public_ip is required") - status() diff --git a/tools/aws_benchmarking/client/requirements.txt b/tools/aws_benchmarking/client/requirements.txt deleted file mode 100644 index 9454801f20..0000000000 --- a/tools/aws_benchmarking/client/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -netaddr==0.7.19 -boto3==1.6.21 -namesgenerator==0.3 -paramiko==2.4.1 -scp -requests diff --git a/tools/aws_benchmarking/diagram.png b/tools/aws_benchmarking/diagram.png deleted file mode 100644 index b97909c5fe78b59d0e636ff73c2ed3e63a0be722..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40790 zcmb4rbySsG*YC484I(YlNFxo>sibr%5)z7pq#&`8*a%1qNOyxENJ-Zg5Rew66a}P1 z>8`sr$Mb#fJMR7Gj4=z?VNjA287wL_!4$ey4 zHje%rG})DMYm>l8i2vgg!sXBXjP7ATI1<-VVTKCw2~@^Zkh}aaisP!pv3E)J_Rf%~ zz_O+XXZlmQul@6^|6AN;4cVLaPg&sa$)Y(t=qj-Ped`5rG{?c9NjE78E_>E4MJdkT z^;DSPqziX%O}gCscLA^rDif2X=S-ZhQuRN6$QX)V_={!ls2qv*v1(Ry>^=`KIi~o#83t0Y>}*~)3kk%i^Zp4Voayp&S3xdEDwNImJ$4v! z*Z?m4Q1JfWl^!uO>d@23R{!16-+d+Fsl;_fXw#ur`=8H$Et_9fr+EMDuRus5@ck2K zUCTGPHo1jk|31M&2@BnxPjN!f(87IXd=~7XNf<4bGl!VUlcK-xJxoL@)cY+dC9sOZ z3P%jyV{Z~7JD~z&8#63t2&Ff6;R<^I%OXNP8?#e{d1GacG5WtPNVQl975VTHV7oiU%ZInQ1HB`R$kzu0!83W|Rgls;FIZ zIrI#Qjc644fYm%qvKwF~<2=0g*M;RUa~Oe@W8D%a@kbJ?yl7DEwlB!GFCV_=$t(Uz zL^MFxJ~&V~>W+6ikeZX5dkOA|CHI~GB3W%)3|Mgp45G1r#R}yz**cf&G z%zcir{c9*Qw_u?zg`vf=B+jppnIi^iTiVV4<9K;M8RL)s4J?; s`&DQXAP#`C|z z82BDFG0cxt`#4N?N%4N@Uq|EuQa}7ytf|Oo1^>28C3MH_-?;Gc2GC1d{hR#Hir@t6 z7ma`ZBPxqHunrb8$Yr_~Gp^;=_o5h!eJ? z<$VFd**DNO!1xFRT@q$J47AX4JP?C1?R%)42F38}modB^ii2-bBJ*3SbzgzSScJjW z^D=zJ`QS{%SpKHa82peitmf%226!$swZ!0R844nBCHf@yDHt!#ROME<&eM`tpmQRW zxnBzVJ0`}Vt8(s8nZl6LJ+$4Rgu&wPv+s2=5tD(|49}#EqTd#gO z37vi21SG>11WPjxX4q?4f=5E78n-q6>T?)mlkqoykG6Gj;U?sVF|o{7Wf^&?P`MoN z?^0KKnU{2CO`7_qZ{bZu*Cw*OY4~07*h}!{QJ?%AA+C%cRD&1es}=CzQz~( z?{1h#z;5y;QmnU zQR~VFn53_PZMA6>(W(*yL)jWvc#S>u&5Z3cH4aEplDK*g!67Rs12dCwe98)UB#TyY zPAxyC^!LME26-BrgxJ3h?OZ41-4KZiIz3%`_19`yNYGx9VqZ_qarx%wE4vha{lO(@ zFySAFc6(>No54UaCX88Hye7!iTwb^1uqZb9g8yHUS1(^YG6n?J&_6>9jP{F1mQNYs z8N|^lo0wJDBrtio!ZMp1AWU?mgB?#5%3+7dHK31|AAL4xv;Qvy5!PCuR#Q}FaG0#0%QI}GD|AXYgCuBSmrs=NVWqM%lnFx)qI0|VOUn$P1wLFc zo}Q!H12?D|WBB$$|K>k5U#aAv_MBPiC=; zyDMA>=+v_Q!B5ggamd@&LE;ow?mn~w>0>!dBPkL*3!MGnsBx$UUPpz}g_dB~F=w#V zCK$AEEWGAkDTtr=3{);We-}jsIbzAZP%?bnvRGF1tb;n#MVY^t3>@17G-|k`<)Da? zhLhs`so0YrEQR!8H>w4Bip~E}qipI!BJ(PhpM1dNFsNoOnM~f^K$M|a2 ze-&b=S6Mt%+S#b;?8lk$g_NN*Y-HPohF#N(5}64P8UO)Iu(PC*!q6+Ijudr%o$Y>M zSqG%00pdv-2-$zGDDN@BHG!FWq`dVz&Rx#4@xes#K{5nn@fy%>#jwg#xQ%V-z}9_y z0y7%>{3es9c)JT@cI17jFxp|OEoty@L(-!&(-^^iM=Pag5w(jY_16@=wZDrrec$9$JAk50xi{AF%<8RlZ z5-}ozty{ApTTg{PRVtwEj|qf-dU2s@Vi+GTI8B0{{VwP2-FRI{rrM>OlZEpYLscLv z^K8WVwGT|a>N?p;XBa3}DCzM)_bujG?)C6HDQ&m(+jdax(o&;!R10xhSVP~!8mw7* zlIJv2lXT@P`f$&$LT4pf@aucl7Ab@$<^xj9i^kmk0|z&hKFMDc%0`seR!=;yp~s)V zIr&v^e~i7Bi`&A{kL{iwT0U=8xtys&GUh7u-xX1xwRkyDCk0!M_{c@b^9{bJiQsYH z*Ep;D3ZAc7Z6qI*33W*(R^N`}WSHdsGSDsYU7)9lv;TT>B#oEY%wZpA;h=gxUVMat zIPQomWGC|)6Urc_jK58@=4UJKq8$$Ueh_xzez zbPZ`G+Ba9nH9AA&T+q>N0FgM3j&3B*s&PVoX)(?P@A-pM{Q@|jT4QjSr#H0W3Mq;+ zcV)*KCfyPI5{9o=_orr9ful)2#>T6oKYRLtj+Nv2mH58SAE~!B5n}mzp)TLrZB8^2 z_@?Z?{rGlEqE9ur9@P_S?WO*$rS5J#(-{_*PjF#>TKR*m+W|yZEI-9j4`7K_2rjUW znQTY1u@|y0OlI-Q_}$ zWBh6P>z@jnX%FnxLR>Md4cBIK)7%2*aJd?^Xp()~zNT%)nj}dMxD9fM?+i?e<}OZo z9*ZH^$e#jmeOD6aP$ZKFuYaU#t@@V<$It<1y>Et((q6{1 zzg@@F{_zd7$16L{Da<{it*xzfx{uY`*Zu7#x2t1GUJ@Z+a^wqk$np9Ug^Wky%sQGk z()QD~X-b)?N=EN}1-&=uX70qYyZtVue zxL(`u!j+PJn8nk_Nt8aF;XhepI<~@9!(a3Ugws?I?tbLGYf)tM0s?=jQmaXrvi!VD zl+n<3i~K!ZS#L5lP|I{k{$n=CmW%u35r0@=J?&^@z5ubl9)L2_AOIYkk$*<9u!0iP zPopUo*if&KA@e0HpCJJ6Y)QhV0$F#)6@EP5wnRprivV$53Hg*4fE!7W04Au4L1pUS z0bDg>!nNc`WBn-JgNYayJnEmsMwJ|Rnf~v3pc4{;)$R^Dy<7wS@=Fusqd$6#7z&IJ zLEJRD&NM>@OK@wtO?L2Kzk%=)(tofq%m75&#+Ucf!{qwFi%Es< zC`%5laJ~MeZw+i6d~89kvC}{!U%5rh-Oq}P5id>T3(GJO`<9KkEVyV;lB`mX-q20( zW`nKHX4ihF%gl!aZ?+>0LCT^N2g8mF6_Nn<90Uvow$J#K=vAS64*{@IPy%ubniliG zO-Vo=4cnESBgb|w-7CXJzJn~WHgS+@O`^)oUMe8Bdg2<(gVdX+C6{BuC69S}7leyO z1ShV!5Rw^sjUgs$znlvl!7EJ|h4f&ofmdMrfMd{qOVO z1UrZM*TVP(eF>28u>)%(P9D+)dUZM4;Gxq_l9pGXT0CR`swu@pD?h)?Q3_+$RM`ER3DMeWV~#IZN`#Pp!X0$1J0x_dmP2&jNZ*3zGz=a^?}aT z=9e(2jus_#pfQB|&(Sh%%7QN#;W5o{%*^H+)e2hB4S~g%)8{{T%yMsXxwI>p#-J%9}#p zAt@qr)rn_eSBY}}&@!PO6LDd@#WKf1j?=9g{J31}G4L`v`J&efC>@}xP{Ji#$Qq-L zy~a^<^7slw1Q#Ph5!~;;YVk6vtxPYj?4S};HnnRwN^c*tnPH&a0kSeWn)9o#Pk5M;vY=D8f2UQwTQa&WbVYq&eo?yll>XM?B zO>|ROz@UF!lnoT7fD#2F`3O-6g2WDjYjvsqn{8;Ub0U$lp8!5_Wd;|b@9LsakYC_H zwm`?87Ah$M^f*J`sZSJ%x-M|#SfN_@p)66?s5aUb{v$A5FlFt=cVGMD3catwd}zrt zXOK#=Qdq)#?E)Lc;rDj4yVL%eU?dTqRAM?O{pqUWEGI`SjNj;72B~CvrC8unG%X<7 z*)5t%Ma=UXS&>KGFUN`u4x(=9)J%--Zc*m&0FE_mY_5-|o-P!Zv|(i?ZDy|{EW z6>1>QU7=AIeN56Jw3&9so~d^4|9*t24{_YynbJ7zihL|{EBEv_Or0o;!8CmP5o87DpqINZc*&H|qOm-#k{eibhS)y}^2VF74L%N6 z7WFW$eI(e^%1)0h#$Pa=(?p7-g!J7}?rxVAv_HD6?zWBna#xOVa@auCFlM0ulDWEh z`WkW;LB4>mV`v;0#fjF4YzrQkDxxF%n>Xl)r`ecx6@ry1>PW>%S>_}7E4}$MUUl$` zh;nRS*&By7k;u#X+OF;$KTgrQ3(=jz<=ijB(Kt|JGu!(u1r$AaI7a$iTAA=b!|lrS zorf>R9y>sKB|=}vy+n!|F%Zq?UdCS#rfhlts1amLDbB)&nm9R zAwn%d8BwJ)W;OdpaXW$3GHJ~lYSe&>0~?eA1m7Q+B_YsHw5mBVvKQw3*i2i^kgLZG z2W5TRjft$W`PxF$!iU07!nb33c+kOG3@|+|rNkgn`YWiJeSYc?X!&Qa_gulTw^$SU zQ|-7kPZ$?!1msICANX7`)0lZBVekye+at-rkQp}$b6(5y{-uI4?-`*X=0qCL+?T3V1d6z@>LCtnh21)gJvONb&$p2z=$I&r?@ z@b&HM#$DLw-~PSVCJ=df48E*DzVYOkho3c3YC4v|doQ3GnCFD%lgu$&%ZIw{8b!w5 zPjk)swq4E;;t6JlJ(s#X1kD;39(*)xY{iXlyKh;Kfm}IPc+y*Z8{)+onz-)}%$1*) z=PYw?n0TVfnn*Q~H}XfLM^_)4S^~G1Y!?%ximgd6wT2LcevV?pu8;xrng&iqw>USr+ zc-$HBF-rD4#I)IKvY4TG_q$W8MQ;L6-TpT_huV|f5v?z+etv!FlH=~F5jR!;?Q?Yi z!WogqF4rE89K_wU`lJZGf?b~#`pzVU5#WbM_{#7JBh`#uKZ zkug!-{LugkRAzk~U{22fHA=sPH?#@LCV$VovafLdQB{Lk0k_*c4Dbg47uKHSXvZ->a}_$L>oKz%Z;5Y+TH&bXicnZfj%a9nG#w1D@djVwjb$YOC*Dke`g3`APruoFk+abB08xy{o8Z*2-Fy zigfcg$Bdn3!Sh^%|*#-H@Z7~aGL)4b#1(i0iTlB-oEa1yH~*X z)N{&p@Ippm_F%V2HU4I&qYUPz^{iV4Vv%Y`yWDP^m(1TfGXRIEQ$zHJrM=JIs6kA| zhIIzQs$u^eoJifl&lehjXJFY+Z*_8=SMn2A&-y6eJyb&DS2a1mjvUrPccvL$SedAo z48aUGRQNS zbJCZG{90-}9`Z|QK&%~i57@_qeOO+(-%h`%PQA;Mrrlw+i5dqQrPs8fUVE$AcVpRX zMvEzAj^@dvE>3sMy}F08)N8dgB)q!UMcK)EnTfD|;+OhL|^r4j)$=_mNt^NlLzK1s)>t?ZcqKC>YG*szY_RviWSZIcf8-p-G>ZblM*%qH@MCY_l2@j|h?oVCY#Fe4I%ft_S?i+TCvU;{^yGe}sA9dS4ttPz+_VjH$O zqAA&*7qa9u!uJ}7XS>TfO~4717`0MvaaN!C9Q?3W;LrUkl^^2( zLZfYV+UD$;pZlA3BNW96|7;FSbgJh%1cD zLw#6gO`a8>ORd82TDLvr!aVRb@a@BvqB*SRlbI59mZcEe*L7swR`r>O!^t@x>8f7; zWc7Nl?%ABKqIm93-^!oie1h$zK_>-4MZbENg$Nryot!`ioq=wt+^@8$L`1*|g)UlW zUJ|Gm6FB5*Ti;53iP-qr+{vf3KBCk^Rc#%Y;hw*AQu(C!{KXWd zUiHX0A*|q3O{#da4~cVqDYy% zmFd0x`MJ*$JJ!l%ZybFK{Y}f^w>r$tnTV$M!fBvVufp6_?la6-)xb<+no&B&AFMOJ zGNS`+NIbEjJzgOIvk!ix*>mO!Eb>Y&2x8`mh+?7QG#$MB8h!~n)1=^mTck=elQ^G5 zTnCd=2gc=ta#Gn-%S}H7`D`js$y^V_@3N3V*r?<0MMtU^%_hwTUC4TFnio4z9c5%R zm-N=``V)yrkP3WD)|Uj>suh&32fgauS4qT@ks~zH8xa3cgUV7c)P61tRl#-%-Vnuv zY%?BFgqKRak0J4ORB_7z-KyI~k5p-@gBUf+Xp zE?i^?IDVbS)}sHstPsQD@iOQ5ANT+O-SGGXQ?vk#eb^w03)L}Q0hto;um6Ek z40X=QVEfD}1(HYFNiowDFn)?6-Dp(zeKZzy4hO8XvE^tPZbzb=Fjs#-;X7{%GrdRD zXVPfSg;tO6DJ>_APeI({oe0hli|!nnUy{Ql1+yLqrh#H_Rd=`+eCEgMD8`kXa*Bne z80Ha-b!BahgFA7xwIb{18xN@dDZ^dRWq1oT-{OR)klG@MgTjZjj>f9`2pehSoYv~v zetQm69&Cm~2@v@9&bTv z+miHAWz6gYwreX&A)j3_T3W7Exma-D;N;zb6^D?nULosW=JLqK4p?o20 zRx#yzcJx9upp^Nplpn@4!|DGahvOZqWg+bPd=cxw)iFFdF0LNGj>-3%8Xu2|cmm)(`he(dqFU7XExEGYW3|O1*ORp7^JHZ{pT?%-stH#`$ zd`1t^&;19xwL*GGjPin_u#yJppd=hKdK3}R{G_^vaa?VM%#m%Xz4<~JT0l!Mo@HVO zC}!oXx_H^2)F}0fWb5xa2C#W-=fdCDikB(T>&VzFVs6%%(I(KL*CBQScDR)Ri{yi| zlVuTG_w+=x=tyJvGQ^RlpLWl ziDP&UheqEALt&_@^lM<5U}-77FYCwbjMY1;Fc2jf3$m$EoW8{@Qi^sB~MqFC>?Jrz!fD467~;7<5T3l!jDchm+ML zUn?$8AGE6wC^q#)wkt4G>{~c;vV772k{hVDcpvdQPKl9oXhZ!e!tFC31HLqg2PFJS zsCeAWc&E;YE6dj{tUg(CRDw)Az&yWFK}<0KC7{ zI+^XRPm-x0(aZWu{i$D~A6ZYf&Jf7SEZS*)aiG-dd!~Nz;bc(u;=>b+F2bXF-?L*H zh+C`e(|UnTOJFY~m%%cBGx4$$C=YDVh!*sI{2d|BWKAr{&swmAD`y#iWzMeGbn*jf z;%Im@^{k8Tt$q395l|hwmcH`)i}`g4r}uvhKaRucROgpz{wa*2ZtZMCA+q5Kq-(vPWl5VA!c;4;(q$6$a zbpN{(uoP@It&E6=jqb_&>WQ2tb>$@cla)`}n-6~4VfqGO6YBm>5_`t9|NDC|fT|Y2 zx)x{6e>AvRN!0-K)fn0#v^`KE;y7hvYTV*oUr~D}2%pWx6qI(%$F2)KL7@0~^0fV9 z0FK@nm~!hMr8cal9y2_xRTxwMrvY4nZgcIQ_B0-n&AjjY1WCX>`QWZ5cg@7z*B{^q z9p)&8Z_WlYDL0nJu=KT^IT|Lb-F$<=# z2-J9yURTHr0bkRQJ=2q0=b&;B;p8Rn$lgEr;lYsfeIP_sPh%pXMq4TvUlDQWxgK@4 zdE1}loL<`N`uFQ}3?lZ|d*V1Bo4x?pIpTr!YCol!x5cm}y3o=!Uv{+;;jg4GX4awq z7*=hz#|skJhw^C!;Y%alLht6BwzvI|#0GE15S_xegyu41toxE;+HAJpu&PB%zi_9O z^_@~EP9_+*6_e`N@Hx+8lWRmQDe7Ua5+QRrXNLTv_?Q)dV^cWXGNcgdENMs6aEF;cg!yjc{(;Ok}? za2eo1S9Z#+gPQR!-W43@tH{Q)`HOOkcHJZfdehbMG9sn?LCFoNSl+f>cEs_ZJnJqM0)&RXwZE7c)S!Gx-C}y$Nv1BJC=8NJoP7t|Y`n(b zKh&_4(dQVJpA!&dpy#wc#vd_HqG9Nfw?3C-Aiw@4Q`qjBWBtOl)6<;`&8a!7dw_71 z@wO6i9g&(P+fLx;k%O1 zFLw+?y*h1-hi@fvcswb#hF!-uulc{9p!1i3Jh2fD!Q8PN9=Mj=KLF)=hp)Y+&+sXZ zdC>p;eF}fA(r{#5(MZuR)#Dz{2s;Nly)KT6GEMPpU$`17KKM>Jb`IfPlKp?NoDAdU>-b+^ zJ}VkeX_{F)_eC4m`;p4wVow-$@o}3R^Hi>55Ah*Ti{tiHuyOz&;Ccls6L;5 zNaB*08E8#dzb8>-)ImG(YF`guA>J(v{_byCosmx+DQgvuD51sWU)O32`S#C=7-Z7! z#Pp@A+l{}EGf~&k~<+qxpco#^Jjaubs=g z&1@tQeZFdZ<=zm9V;A81`guMyaf;9@=19?h6)wPKVFSj~k!FBc;P$@(Z9{eKla5-G zsUs^~pf=Jue>!$ofW9FoJLd-fi~aRD(Vw2}Q+YkTi6ZtB<_bH25qqxdG1nC(O7i@g z?{wu8H8eM%P}3XF6-*>^h?*MuR-i%Sz57L3%yW}5Mc5A8w6a6R3!SDP{8bbLFI zKeEoG*-IGY6#sW+E!4e+=wb2uTlIB;M}Zafei!F;ziYntvEu=J(Z-(X{ck)u#BMOT+gw%nG;n$L75_ycR=()Rdy^ppRXY>BTqzILz07b77Kmflz_>ycotRabyWoRYFY-scV7x~1J8Dj5r+nxiQv3j={ad_DLDdK8B$7zf(^91 zR#baW0u!0yUZVNUy7kIlb%ms(Wp(jv4(nBd+`ehQ3!l|B@-_g{M1CCr?oDu7Z-|Iq z>N-Gd?P%(&PH4D)Hvy!LnQwMwpU{ZJG*ig>HAzW?Dv?ks3xr>hr@%w&CC@ zfrZLmg975$hY?Movn@2jr3L8wl+sSICw6;WPct0(@1#72=FD9{Ar#SX-K;1ma~`M3 zJypNy%V(--GqsGTp2UA+p>d-{jbc_^+JYMElf|J`U#8#rWS%$wlOPhaFcth?(Hw_^fWo%%h#`=TNJ=)o+=1`azMj1MR6=~J%v92#8-WJ zvp12zln`T65UMQ}+#%aK{lPYNtVz5k36+gF9CugcKBW+ydniV9K0s?cSNkvr_29t3Z;Xn)1F z+W!`jF5%V}0LvW&FMLl@2YFk_tWu7g} z^pzM8U!rxLp9K|ED2UA|=^csm z3=fGvhDi2WXZl!ecOK2$_8(~5Wn4_%vGXbcI6wd3%S1wh5h!OLVD5G)sNKdy;y{~n z){cRvKiK7%G|aMz3k7pI7-q>BES0Wc-bus}NPqdoN~u`rCV7R_-M}`1sA0&A55xn7x&?C!dwt#t}T&D&DG3fb*JDTs(DsLliC^~{hzHnJ8(bE+Sq;>T8 z)nbfFNgn`HK0crVe8$;F9Zb|zKPT%v$4tdbY`AE(DeAD?dLvhAXCi?KVCvv z2V)zE-qm9KCQVof_ehU7b@Le>p@7g>Q6(5X0Q>1KCV6_e{q4Hztdj8@Hyr(&VJA zC7jTIYp^K4yhY@n1qM5bj;~1A`*G+F=n;6jnwFV1_i-O@^BLLY?o9CXS-J z0fZw%xaV$0f(e%Mtpb^!ue!n&J!{G-gT^0T9L_~~{t9KMW#DD$Ss4Ls8%n(ksqp05 zldZ*$62mI5Ey?G}Zc{fD`}2|_3?7q|E29W?&he4IA?)u$x2w@5Fk?L+KRMon+n}1I zUFKC(PxgQJKcSAm29w(eE=E3F3p&$ZUjV+~eo&4w`S~$i!c$n(i2;DR>iV=i(4<_< zAyWF`@n_2RTx)jFt_%Ugb6TFur{iwA8&$i$b<;tq7O}q#;&1Bl4)G(?(Ee0xfKS2! zo*SK->EaOkeuY71%A|aIr7$xkT*Iiw-j-LlemN(+uXogsFXkJG~BCxB$P zbpE+Nsv?haxP4;=#!RegHVL|ij)0wcu6TsbOPHcgt&j0>oJMk^mxAcXSDP%N#-Aqg znb9}lm4j@}<`C!u8cd{ntE2LAE-LgNElTtzatLW6EOn8utTYcFicUCdL;m(R&4 z;v`2!BNH z86=J66-h0uj^FEdzQe$6)-pB!S}R?GLehPmKS$C#m;I#VerZbRfY4q6==Ukb?HK)z zert^>ElpEt)&G>~?A`rR7x_4$;3Z~=eDV0~%I1bHO2b1wZUd%`j~awoGw)o^hfu)1 zMm(_WoF)5J&;tTfdG1rsM>NQvRa^$!dsI!S?CQdg&J1miJvAqJ$#3wfC^z{K`$O8%)E3ej>S)~Q^9F!$ni6|tN$wref1$2JxCr0 zstrM>zgy%=`N^4@{yu=F$O4lki!ZK)ZM=Wjn9}Y2);WNE4fMep&-T9y*bFhovJbQc zV>vG7X`G7MPn3T>?(?yJQP9KxWU*oGy-!yBZZr?V(ZVcH& zW-!m5)_3v=tR%2Ptmlqg4hwHs9&s34l1boN4qjLY_Pr+;q#^T4T1w@~_uKkosp7St zs(q{am>fJ1cP*o{O@;%R{9+)E9L-o9p~oOzS?5<>fcc1&%$@8T`o0%u9wS=&>ysBx zj1DKuPXGE)x4H$B16$i?!SGF>nVq_)fxCek371#mDB%-|w%gsy+F)ge!QT%5j+nP%$e0efqg-tuPMexVipP@WoS zTo5p;U^aj$HXP>Ot^iBVq|(09hgU$)XijrBU?hhltIpCr4OguByaCREi)?Mb9P?v@ zuYd;QcqJ~re-Y#h4M}rg7e2O@hnS%_oJ}7#-(C3T9i&hE%11VV^6uDwX8~qDa4hBD z#({PTM6MZv`muIYR%8@}Yld&v z!nh%R2|*!!V{`y`0}?o5yU&eSdf~U9lw{1xOl3pCe1Adwrf6EOx<% zoQ14d8B`J6=*9LyY5ILG&*&%FjQ_7Cn9a)B(Cf$tB&Z8Os;Duo-zzLrz*QfmV?~P@ zgz+0HasI06b~y-)hrWOB&n9%+Qgy?^ihc{)oHdK?iM1q#S^j9261X`w&@6rsCl#lxEL-^32bT@eG_dANEoZr@RJ-&(R8?bukr9SxUk}3t>s!rkN0M^%BjH9g zQb$9@ms6g9Z+k>~k{(8Z<`sl0+`7fmSuu?nO(>Uo{Y1x=WRn2Kt|%@lwUf4Ob6{{7 z6uJ)fMhr6mlucupL3-v(IUMM~@Mta=L-l52{QGS|q|Vh7R4!wtOP!V*JKP-)tcVbi zjS_XR_i!FX4;A}cqNi;ms_6YPr(xwpa$Se1q}5n%u!!2>do7U-i4imsoM9&XS`XR8 zVC>@zQFpkpvV2|&#lAqR23*4p-A>vpdKQSuTg}uPg^@&VVJv&kj|3$Qt=8Mb#pZM? z3XQtGBz1NXZJ%;+8=#87W_6vbosSPmw6IvBU&ROXLS~5XLUf|Cqtz^L!_Y#{hOL91P&YRa z-RwDaayohdMP$x`q%*9vtp9Wz=#)#LFtzaM`zwWd$~fUpa|i*OA~7cy1@k|vV?60F zcg2E}NV$(Mv1rgEYdyl_COdO_TGD&gpiJe%bpGOxfy05Wk*}ut9AP0xTY3<^tAr+&c$ISDuKtC-|zf10CfMv^(!qumx0{gW8As)Liu(-zaXwF)y}>X36gf(8E!S0xmnt-cU{`o zbhB>zcvr`?NSAf@_jgJPu5IFLm;FRsZvR$nLh28Q!_C>kjUSB%Hj8=q?6@l-`XPz=Ej`#Nl-1ZJ1MUMoo-rj%1dSnSj~{N75*~lj8lSHWn-bD| z5z`FYe)K8K0 zO3G>k4nC-;OzJLI4wDW&3GclDBnsnl`g7YKsoMiH7eJCgufWJ^@^3OYCnu*N7(zRE z3ZSiIhPzSLZrzRLI1aOlM_8aY8w@5LPb{r0FE3u#Y0ZE-n>voqOol$UJ2%ULGSub! zTYUix_+y9`YttNwH;TzmCx?LS;OoKa2QYs%-un3g&_>(&kn!EyTvMM-jJ2o!qbo3) z!wtU+ns_?!>so+*ZnHXr){>trVtZa3^eWdZ$8kbk8uRq|cwml3#+Wk*=rK3|%DgE| zq%zG>5ildIDvOZ>U=dQnPrtqu0bR)EDJwFw`>v{SQS{Q(=&{T1POZPyWS_)ca`ty; z;tfnEE5Pt&LRs_vH@M^+NFUjG61I3#S7!*h9zJnJZuMcI%;KzlmIFk*j-7QipA6?G znjYT;meMP`)j_5ng#Y}RYTUuvI8hT6x)k>=1l;tW*-);eDC8wm6LH-20A<1nC@1dm z0WVxESk#N3H#STr`7$WP%nzX`AZn}oU%F&3{>EjQYCZIY*Vqh0p#DkNCK2!W{h4sR zoTe8Ce`dC#el%>I^H7keT!Cwz`CECRQ31;{x)!hIj@_3GWbW^_knB+ATt{a6y(hb=EZ zrg>sijvLnk*iof{B!RhC=E*)!?b5x--s|LI)BBx1(=RiXj@j%^WMQBAx}mJi&z9Qi zIM7RBU{W!6veCK})4Q^mq!!nBS61NIw?jfP=MnNp{nu}C;M(Y2*uH$hNrc?bCPeFG zwcYrrj1ZTVYnm5X%VLTh|Fd5TaX0VXv`YQzk9&A2y6+adkmQ^G=|0bn;~hd8kq-@^ zjXW&PD4!#BzajL8FE%m~q9-g*?Jse?L-l)_o>3P3?u4Ds4WN0J=?D4=4fl$47wj`r zsVfkg&3I-mpIHwbw-i@i$MBs$Ywq0*H4v@Z3Fvmal z6L4T3WG%PXe7c*q?VI?P8V!m;Bc+x)&y15ph zPv;2+I_V4%trzE#Ef?o6KpET_bH4su@cHskl!W_w_d!(bd&vZjT6wHq`a|A(%(j_Pvlx<&y}LP0u|kVZPByHQF&1qCFeJEW9U zkS+m{E+tgDLzHfQ64IR_AuY{Wx6k)J-+Rs&=dZoT*qi%z$91i1t-0o$OaJohO|&sD zTl1kDk!|MFU6fREc|62cdApz+p?~wyBv7^y-4b zyk7=%QZjz2)Bc2=A$C~OPNPz-zeB*qG97SqWdsPQu?-;H36cLEG81)c+i}n^%~#TT zh*f;+`=u+LhuaH{+R4xejs&(W-lZ$oq#@du07N=;p^+{*Ret+H9)V&wg})N5H;IPS z*&TK0xj+9sgd<=x+`^9%f4#ppvH%Ul_Km7tYf>ZVTQJx5jp#z;;@u}-tQ9Z-``7d7a|z7sczC=&^}F~`l9{O+Y7+Y2BG_VcZPegcpB!3knRy-V zbq)+%)f-*#9Ri@BH-dN3ktQJqD9I}zeHZ}%&g}fvYA|Q$LR*~Mpq51L%_+B7e0tmV z;m@C+b&`w1({w&Pz=254FPjqMTKVVaXL5w*O#i#*?l)j}mzZ^v4Ps@l+R_FI6q8A> zay%|g$)FC6J6Wx2eyR6#)&%W7>Lio;X2}QAB?o+?$Y)>MzA=0(YVpNCH){kA$tMyn zeUAnmT8Fl){_rSJs|nJ|eA~g5^3x_nzaVxrnL3&2_T?yFE1G)Ie|QeXH*LezX`1VDsJmlJweO2AmrxWRry%EZcLYqk*=xW6>q7ICammrcJ<^kHq`_>o^)$BO+Ss;7B= z2^eM)gBAUQLBvUK-ws@t)}T#V&bzB#kFz^)t6@yyVDwq!cut@EcTNoTFE6WXU9t`9 z&c_SLBO;(+7mQgeLrG4^mxw+8Wi;@hdi!nv8f=62d!(Tj%dhz>0_l?p?G+myvyLVn zIJ0S<112>WAvDdi`aC@20``dFYy!JZALjM(N;{k5NTu&g@b!I9K}gVwcKSit^Ui11 zQMValjA1Yx&5T z%L!Y!Wb?IrbKy6)HHC$fju-}B<0ecHagbZ2IK8PsKb#NaRs@EN)6a)GbI|&e1%Tr| z@cKyrJ>oa-;aO!ptt+)1<3q?^&~)QRN3+M}1gxVr#u@j7?cpb*xu(F`kLHv2$Fr5L z6U*eXu6V$S8q%GL`}M{C_z7Iyxp{>(WXJN6rsKr(%Z>@IlXKgCwT*hpWau7}UhU4( z-Xig1U^g@NA?@X6jUjxo-52M^8hi6C$&PS|o@Az(^#}5|urX6~|H^Ffn`RsMZVZRD zxhr--SIS;p%D4tvZQ;V^4MGaL(1W#ox`K0?WP?tVCM{H}+R@nSXej|6y8WBHh|nu% z->a>n8n!hT=O=2uNM9r*+z^;l8F;5h3giZHHxi@T5-cY;p4PwjWwAgALwHPG*&iwD z-L%oyF@JD+LdIRrFL$ofxL0>K`h49Ct$Uk$iQ?D(cPA>1Rd%O<^ywdv z0ewU7SmiI%=WP!WGN!R3q6eSLHC{Qx0ZV=B<=^?>$;0q8ZSyx#VfW$vh~$)qR^E8I3Q&R7zm23e96?9Q(-C&mbAy@6U47mC+zw zHJ#&oUofd{0n|N6g(I#muI@B8v*nn%=W`iQ7OWy^wF&(_h>PoRc&6P%MH*s-T-%QS ze!u&TsG5Er{9jRh}YdQ5Q2kxty@w2(dY1R2J`zRIlYtKV&U;@sCs~D_Z;Y~ zQX%sxj0z$9YSoY$%(kz5eMy`giV(`-k`2R3lC=iDsfFYGcL)CBRDpwNq{qR&JKtxIXmc(EfU8zFk; z=rSpkz{3-UBWy^|-DS(kYA3C&x3TBU1Zyu2s@yLs`wG9Qk6gXM@@UgSTdh zNoktp4T+Z(&(q)?jCqDlZuBFUb{YBH^5?9Jy=XE^+0${k=C)tsuwutXBkzgfyFRS% z5&7K6E>82p)MD(~(0Bn=9{&-foow?Sgtwb_U`+395 z&*cm93P#yU#CsZ`PnszlMM31_ zhGid`P|i=Oa#LbMqpd$%DS|@Ec`^7*@zZS#E}~*~NrO9$gN6S-l18fP_r~YPq2E|4 zKE*FL4nBN$GBKq6M2PgeI@%P*x}3P`h$714KFUZn#Q7w4yZO3Q29)-*c&=>pvsf5z zt+}+eu1#0!&P5GSI1{lo*X{Q9{;az&tZ<^)9~6ZoK52_O_RQ3TK!`A~5gU{fOEvH# zJBckZX~lIRncrEGiDVl6t0Qox@xAbH?;Fbjl4=a@uRqRxLJ1(~-<9$AmyHDco}S-9 zy?e-sChkBD!SB)4hxb$vgSscgB2M^<0- z>896zg&+0p^~SMbV00<6IO#4A{z8fCS6BB$ap)e1s%D^fXxE1VI%kPdBSt*{>4JKB zN!aolMxaQf5Od>rQf}!FeQ=~ZaPhmR@GyKD*SF%|ot`StMGu2Ls76kNkc%_FsV5qi zmx=ZV^F=s^6fr2ste*WON0%Z74DG-o7x*xv-+ZNx_tz}G#aM?*$6+Fz9wO0OgRvi$myt?2Gtb4D zH@m^onx&X#T`XcprF0cIxf>dae>bOMq^sA=fHf&We_Ur`ulxQRD73FjE)^h-OK~Gku}t;Vi7%6h)5eWwJ4ZV*H_aYaggDlh@L0 zvj+a{W&51iGSEsM_ME7og<@Dzq|Z@P?n}L+SZQw6S9DQOLa+4RNb@Tk1%Zr#==E)B(vMO6GjPIff(8 z_I+?c!9P*J<`FS9P=t2wK3oq)$tPkVTZ&uru(dx*+kS5yJA(sFQK9A~eFyniOH9dU zx!tlFeY8jDP@A~wC|zR4IK-32N@&;oOFzqwu`TKy=#bR;=&hC9)6!nfGdvB6yUDNK zKE<5G&j#{N_Tw^h-AY?Zb)e1p3Y4%*ZiJHth#~>&DZ%LBBy@T`FM)L|1%^?9YYZND zndUYNy5yaB^zSZxVwWs2zpv_v?L4-~+s)~BKMkRq`=SM2fH`-b!m4YqI>vsengf+S zZzHbs!XBna^_F__tNC{-ot1@1^pdDr=6-yv_Gd5N|51a>Gqm>D%kmgUo>%-<(HhXV zs^xqHF7u3v8k^P0d4Edo!LiJ?SPmEeOsBbtXI)3%c+=+2Jx0t^`tO5fB4fkx|IVz= z7n)PP#L9L3v9C@Y&?GLiZ(*h+sel(sXY%DVCeAv)x;~ z&7N@S*WB|zzqvOtpQ0BNes@fM=Lk_%)T*7C^P$QOjBRmmrbKmy5_Y1u=5w8zO4y7O zwWUblq)Pm_bskP4-pzk0Cun3~ z?PrJPXBos04BlG3lg4L;>$W~>?Bfi0>m7$_DqwndK7S7#c|53md8?GhIT1nn!>goh zrd1&YFUpUA4lh`xS8HGm)Kdh`WV#m@JnEZkH&Nrlg0vRJ_kPG#pE|dZUl`0+6s8>f zwDd;P%GIgf4xXtBp!@$U_stSmiKF8fKP@ZvSy>kSBdz$P^o3hyuPAa=T=Z7FrZEeVcX|) zTS95zXJtX{!QCf%gtXM@{mm-svR%%|rM z^x{LILVT~x8z^{!9m0lNR5ktLDj-+_XHex4156>8tDgF!&_HYu%R%Df4_wmFqP|qI zo0&?+YMGA!62oPB)=68o+7GSob<LQ+@1mI2z(tg8bJhy z(#%#&bc0e%s4@ajWfAAa@1V_p4aoA;;Q&aV{z;7)OKDnJ0!^T3?WG7;ybfoRm#8%1 zjh%E`OT%o%dbUdxqR;a}p6(K3k4w*~`m!daaslO#H|sA5bK26O8Iar^&{3vem(}xsI^mCZ0$V*%i|Hv`V;69eLZS_$vjCdm3?GZJ79=L{N_qxtx+ zRie=?dj>)c^+aMKw1+y?8b#QVgJLDf^X zGn|rt3s%*$rU0B}*H9Tnf!n6H{f_t>6pefSQEVdaF4cj(Un*k}jJYY;*X2D)N3 zfDk3nV;MZi&rRby=vP1gh9Eu4Kw=;VF00UW463kK`_OjFKQGd&!grtc%w1i9r6A~C zMtzczXByfGgctuXuG)_Le0~=hGK4GhELO?Gc(2~9``xlXBax=MziOsxOiz%Yb211k zz-Pcho{th}&m-cxP{E@C-H`%0GMnEl0WK@#Rl@}!p%JDhs7!k0F449e5=v{sx6$7;r4qVH?(GA$I9$ybC7^6P zs5x#I=-!f84<70mnAtAra!{qyM_<21@&-`PkDk&;fQCeDeRNQ5h7>mFZZ`Ij272Ew zk)sk=OIW~0eoNRA{-o$h#qYRu`2dtv^(cObJ)g>n#7bbpVt2xI(3AubF73qzra#)E z`vve<>t%9%f*4lcS%0jaW)dk8;}^u9juHAm2wx@+M4&U?_iarYk{z$EE6oM21zx+| zz@m|sa$GDwr{3TDfMk;$JL*lY!3c2pbB^x3@gBXahf`5?@^bOH1dA_sY&myXzjl^* ziG`Gxj{_FZfas>9wn-i5ANlVh7^2>-BJXm4cNQu;4kUpXZctuc4yd%$8PU&L#KbjH ztZ*Ux^WMffrM!G+5SwBuXV&!Ir9nxG#vCJy01*jIN$5z&%8PnuDnFa%RiIV46UcOu zHF}+V=}UJU4z7&iOrmTfUOrt(--EF7EOArX!AIcuVn@G8GDs)3v)K6>mSo~kIPjba z^r2~IU%Q$!bWC!X*!T8BSK@ zuC9b$M^pVfr@uK-bzPNuUJiHSO_=jRw_Ert7YQd$*o*FpUZ8C&Cj-I~s^;wCpbt~^ z)4X-FMayLw$40ya*1U}1>M+Ga27+r&=)BBXA!1?j*?#@i8P({aH+bI`%e@M%ZkZxr zc-3a`VA4r00+*22so7xl!n3<3*8Epu;bzU0w)Y<$RP&|2p#stUH0zVkwYlmg!m(h+ zPE7k;k7!Azj)i7{UQ@EIr;@*{Vf)VKNB2~v=h&I9=qvAv{(6jEFuz7dpBDPZY>9<1 zJ4RUcJ{^b1q&o;CMf$^d6gUVMy6yP{-Tq8ZfqZn;&4_RYo3%;$%{9t-w=da_Z)@d6 z<~YZ&ByenQnJ2puqy0K$ zQXwW2r#NR z$*X9GW(l&(4@q{Kg^Sattf!@K<%A?C-6#IsXTpW6G>?Z9#L>0~8YdzcC8-xdP%0F({e1XiAABX5ZOIJfy>JFxGSsLA}Vd>{B`<+tL z>jIkDgDZTu-0#IDv{MEK=T>vb63{gfmXpXn2=>h(i0vt438MKsqk z9239%s8Q3P_*eJOP`>92ZVz#UGOw4~-p96}n-lqk zrA0g7_WZ{k6aPFVWu?phlz}G2+I`0I0g_@%*ll?R@*GafByIk7>#rE{aNIL}-1M;R z$*znNM5I$#kIxct1QwGu*$88HZc3dEyGK{jod@7K@L->G()bRU)9rpbpu99bD~@)) z5`obWg=;JlQ1L@@P96Ku(q|GnE`9DiDS%Skb1T_utU3$}#0#uxoo8ipWPE`8@=)Yz zQJ%fzqlSfux~GzLdnd;@zcMUD&9UbrCqxLB+2ZAZIdseL_Q^fZlsR^yd&YQ8iCTuW zBJvz!sbg511a-l^KXnYJs4*C6lBNcLOy2iX;l~JHSb z0>}upUKv_P#gu4!yw+0nXR;%1dOKy2X?GMUwD}@%JpY&-xXK#s;nCHfRi)Uym^i+z zW>`?3!mV~0h1Mz)_{%PnVm-mBqDx^B6H{lJ+oE=u+-D2X46bB5xflF2WQG|E&%{ry zdw^kZzeag0$Wp3mlH<_d@2H=*Q;74-i59v-H@AcQm8{AA%u0AUfue;3WlUXHGe6eS zN5A$fEsC>vau^z*rQ19wdLHc>@a^uIk~ojA@bmT=)4|}fMU^x1k7zv!5sY2;w0@y? z3bzwE>qUqZ-FO}L$77*zgyNZ+ZaQe#Tr3Oc;#A{=x`vH$sdDHFRWsQ9HrS~5I8a~q zNOABgnDx98Wc^fZg@)v|B@GKbNsE5ARxOK7SFe#el%xf~eSCOE1HM+xs1WLp5WMD| z$JNSXJXWu!j0v+;meWzrSMgJ-n+aTpYmm!Irqnjf^HYW0G!OaXadqs^)>py`<9E3y zR@8f{K^H1)U9?_XG70JN#J@ow;;e%cq%SQB2LJ zpVry+rR8WAYD51d*wlF4D>TZfv$4#2DF{0*9sX}2C=PHvQ+Gy_qD&EC5+k! zk2LuTmZLQDLyjUxs-EU=JUl_%gf-e#Srb`(k5_Z~gZ1o17p}&Jkr7Z%8j)E)F1L)S zbHW$9I>yu?l=kg>zi9JH)wqtu%1f@!Z&6{c*jXvbQ47I$X+M5jD-Jz~X`aI@hFjDw_L-;GXyc)fMX@(d`5qni*i*Mv8)A2ozC4e72#>+v{}EO?$k!<)_9q` znyP5>tsAITYYp7&H5Qn~-CC=ckKvFOD6|B)rGCPH7LAzOm%0zHsQI01 zO7%2w*$W%b2h-0Nf5!?{yL}UHW}+Ky47?7F$mbEMlYMQi)T03s+PUhkf+J?vFZdVU zRYDz76DC{jlNcsa>fTioDq_uDYklLS<93e%;a(J{zI3Cm@Q2nV5{?@bS4Vi7L!E*D zQ&&kLk-Wr-pyN$dAS6a`_P0|Cr(?ab`*11iQ09D%@tD6Wir0Q_@QT` zt&{l@=aA7qy1b$8RVQMhirO4$@!pggV}lyI?OYunq^!PiNZSZ|}l zCMc*`m)Uc~6PkZlg{dF!1@=`1^j#&ibaFS>-@LWH+K)&w|8g{4nd@aD34ZdMAp<(5 zvu^6ft!AVgPl_XY2tZ-@p)6u8A^X+rD?*`GfRj*D8A{YGJXguudhFamonBJ+IO;(6 zIgfeEF(jqmLM1qgEy<0q(9U}n)&6GtEAI2rz&>$-6Me(!5;X?2iVyk{PQ^@m_ZMq^ zZ+-MLtRXCcO`+Z~0vt$5?;8ej34ABGKJpz})pxEq_1XFMQ2OZn!k4(p>J>J+;?iny znNBU3^gL?=s`F0owk$oin)NrHt@=y+H@wXW>-A51n_>iOjJMuDi*$0Wv5F34^_hbH z{gju{bpI8n@x8{ZBn9@3Ba-Dy7gBWBBBS4~V~~@T>)+L|d=Tfvcr133`iy@xX;W>HSwBK83ovsm-`~^>qYJ(^892f#@`nT13M5HYKCQ%g! zE4A>Ch%-puUjMRvgEsSY`sJURDcu(AeQ4@z%Cj3D1Gma?;}%i2%@@Mw>x-I)2o=8o z$OxbJy6t(q8QiQ5kHWVU@)#J$jMMK}y(O*~P4A{@3&)l&*V4V^c*)T15Lav|n5^ac zt9ih4-7we=voq{JsR6$*hMNCR zGnL8uX~^2lI$v2wCs#eXiOl786L}FArrOg0Y6mj$`k!;Xm@aQcI=mLI$dYwaeJ~|# zn^Te(2~#Aq^?Fwlv0a++k#m`O*I!7heyRqDN2#kESB}KDL`)j!JFZGRBjOy;Egs?O zUr7rL(7K2}4`aA1@j=b&7&!#qF3Wa2yX$-qztLk`n3#F!V{V?DJk~*<+CR`!$y;_= zL2_m^Dqb~i22vWzfU!qz%t z%XceTgM`GwV7JE+mp0jpiWS5-7x9&2o1JAFUQpLj3@(N#YVmfq!j%(U{ zdwAO?HT~WfayBHt9~1iN+rWiy` zz%7vd>>|RmXX4f|E2Axwbqn?LPoJE{PSy*RETl>)x6DIpfcVsaIlwIG-M*%WN+|m+ z5H9`p`V0ClT6+8Go?(Hqie6jem}-K>U<@tB2PXEVuoFQd%JW*9aX5}!EkK|y^E85B z>gAuW2Etc;%WT@oIq6CKmG$q`S$S=j5w~wrmlUqIn1}5sV-Arm#Yme9k!|1gxYm?l zV1UUj_PplUOoyNmDr!ll@faurPKwbp<5c354(b66*{qQ89pyEPX~cKhc<WMhVyrQ%Gr(8lMKCA zpntTPg0L@`QC?`fncuueS0PUOB3cgraci1ou6Q~t%QSUzUVB#95842_rrRdHCQLfi zEq08PlzN5Rp!l!XKsUL8p2nH?E$XV3(d~x^q|>)}eWs0W?GU!qi*%CZ2Zrvfw-Rrd zUR|}-?6U|grd+1XG;9gj6kS;t?RvxQl6QN* zJbcIc$JBf+dCbmyxTi-L$2}*pR}iX}Yi8wviFlu<@zRaW|0Iw3m>=XPPHBH#kR0w& z=P!%DM16DlwX?VZpJoGyjjqj+VB6zbb&pn5 zSOdw^K?V*&7jg0sy@o@==PmW66E^}I<@X6N*V%~J)#wi5r_5|SesW($YoTirhRP{B zis$1k#_S&yO%}x!l@cwl7~DL#*!n55E+U(BbWI5oBC466x+U9HI>}zzMxiJ!iR}lk zfEKULJcwWPK9&j@MY#-}m(ft3Cf5T-w$b)NflMKi`C(%jic2P8O&Gu0ZaO?S=4|0o zO39`^gAAvdBS6cry>r1Tg1ll zJxl>2E^iH9%C>JYWb%x5n^4JUc#x)}z}u^L>U}TAhMsYIc2s&J| zXO5-3biIY%@WcKRmpN8E_mhVD>?Rw_7VbLZF)8jhye59BW8)i(zlH7tCQ(O?pN3%Zmg#x~l)#UT|Y>t!94TI)Ktuy#Go;@X(^h6AX}@{Dbok8Z*UmP_Ht8zr@Hr<%xWN@o3C z8m7^4RW`+uOtvuV1qT&QUDE~C9#pa%S+7N9z!|kuDL4=+JAs8)1dYC z?@SbRIzXZzWJ(SWz~qd!IN^jbfxb!8K-D_t+H*jE+>f>1 z?%n)L&;LxQUQ$!CLbR=ZV7}8>i%)a9dQfQlYqtY>jc!)Gc%QU5p}oQ{vP{zy%{l2i z7DB@Ajg;5ni4iqYX3K+byx5F$87mDxu>{4E$0p3(5)vgW%|{jqmIgmHKTzi_yV+yw z&JPL&u{oTdc`9)ZTGTcT8vE!!+O{6b9miGMV1I3U`N+pc{NVGRKGd#9Z{9!RmFd8> z5Y4)j9#G`y_+Hp`{^vva3<3bw|Bwv}$=jMl{?#Vkf5>L|OQOnHM?vLcy$&I|A@ZX~JDOj4u;E?alYe{Fhu|0O z%M7MVICE?P#a^qL>1>N>464iQsAIZ^v8>lSZi&l|y_ibPHr1UL28es%oa%)R!?*>?b$C_Wmx)L>%LPVN4@ zYP~N_Vq&v?w=e5>KyH<8R;=i)ETi3m+S{4n=HUciE%Bt6R67o4$2vY&I~=+iwBUC` zLqM3J2mG)pybrr8s4AeA$G1!GYsFxCtvpn~=L+h#y6N!tlvy*4#9>$hKcuc}ucS$* zX(t)~#k)q@z~B(1=$u>v-lY?k6iO=nB=v#3(L;>DNykJIAJJh5aW?1N6g=3-E&OlT{0GpncM=}C1jaKX$qdIAP?Gs6%<^Y3bR z-VC*EzdwQm5eqr@ph!FY$e#(`W{qz&yj59T-F?IZyCOw}VBx@enTNhH) zd*MN59RwxuN3x_3J_#bm;+DwW!8)R-#(^fydq#ZL_W$%~qCSWoUXQ$CWt zO0*O|%QQwI3ZgeYkG#w*Dq=DBO60K`K(HN&v%70W=d2n&22dEaC1bNST@$R2Vr;oS> zXwL5Bu>R%Y)_gc$8{)V=T5?e2d)8f8zjLn+EWe(wS!9`qTw%cIWFVm4OR+JZiWL%% z$V>7&)l1Zt0i5)kz@vkBKY|AzZiB?X42tG)=(}%nQ4AEMlddXKa$@A)A_&>JZ0?ql zC=nZ~nQIeWmo)Z(e}dDXS-r)Qw1Mc|&6`UZ7P{DMgJ4oK4r1nATVQZkPJey5HjL_e zp8bW)zlAusv`Ce|46E1n#k^8V1A3Z~`N{jo7khXH3}pVg5DD5t%e*3hT|DGA?P&p- z^3+O`#ol5O9F{@B>!`mCy#C4x%l@W5(KQaCch!7fABa=nAzlsG7@7y9 z;UYhJt|X7i$mlXN%-`B_=YN)ankIZh`=%L72COxur|v;SOXwom=sC}!GtnpUS0C;o z`3wN8;evELqkJWhKF>43CkcN-ZD(%te#~KEL$N>r1HI+Y!d$SP@c2=}HYg{ff}U{{ zz_R3^cy0LsXp2B2;6sA%Rbx8#?pO^;$^bh)S-7XKnATy=%KFyk)V=E#<_Bgk`X8)! z;5@Er?KylCbE`;KNVFOGR2Q&VIeba%dZE|p@a=Sod!uG0&BgB5Fe*#)w+!q!3LN<{ z9cv*^zr?ejJ=%O^{S2&5Yg@&A0y zd()+6rVj4*7o_7j7rvp#h*!y2&afz?BY4*EvSRk6qo?-arZBP1kvP9z(TlmgxY93- z!>Aj&48->j;uYPbacU7i$_?PsyEWmMMhZX>z-Y79!j#Xrg|Q#H^XiF~8Qh*O4yzZ_ zqZB+FRV;~{ZQMh-AYr!@nsj`NY6c0nti%&QHL{MniUA<7@GiQfPj@4c8aPbf(@TxdZ>k1H$E?IV_97 zQrMc|@6~FcPhh+Et6vcK`AP3w*DQLXe}V(G^iy~To`m_YusY5 zt=sJrvjZ;-0g2UN6p@_HF8n-%%_K~E0xk%IeqPz@?C2sem=)1`V!)nd`H_D^%LE&~ z2#3z^|K2fuG%H3|!&_E&YaX}&6NrwV%2sO*0s$iXD{vn-rM{i+sEEx6GuT2&)H$&W zkg;jY>puW@dafywPJ8XD{Y60>@sL{3CQSZk<~ib-uL5Q+>u~Ot0a`#Jey%lyHY~}?3C~G z!MA3T`}{^B!x)pKVSvY{M6%#@TFHFR*k}-~k7W?G+j|5Rs2sQc{Bvv$Sbxvp;mS`W zE)eK+T}kRVX5zM@cVrFCue_r9c&6J-hEc8Rcspq3e8A6aER=53|!-1iaD;Av=4{V3XZz}$_%RIJ=>1* zEk|m@um0cP0K2++`x>=q`JVd2>+o5o;lyga8P66fJ%%*t#$raMGUYv2Q?~)se++C5 zYf({<`Ap#@!x3rUi$Z8!NSs9QLFD9I8!gF30t@s`d_|1!9XCzb*YzU)y9n+)h59>Y zLHiP99;wj^XuD!QZ+OgePX|=EU1bH>*83RNt^Uwn90rvkV>zmT?b#YBb?yPDO1u6; zxBbp%kvxj>hOk8X$kJB+*lN;5fIM-=%Vh#!xzu1<9wWc~KQlA<;#<;d6IDgsQRiR^ zSYrX)D}4)Ex1AZOlQC3EH28#-3&121PRw66AeKGo6`C0ePZwWNj?HjSMXyUa z`V4Sd#^6-7ziD4>yqfxn2K*H2;dSH+u{|X5N9JO4i|y2kJvfLT5PMB+>Gqy*A?{hE zNZX6OUvqBh*|P7ih_zua8=n2v8E)`?N#5YEtHi^QMwl183zl-@;JarBemKrqrw+a+ z$}xZ=n6TbznCt}tT!vGjgYf#AVEyP!Z08vR?q7M=cLIZF# zoVe+Z{R=kM;D$YeFn8TLANfHezTm4=>Sg_`Q*a>ByZeE^9yZ&s0nHHj1F>Wg{wNO| zMFAwrQ*r@wVYHR`xhXjI!6*^BTkExkPOYTFV98z8jEX#i zR40Moiz%H|NgjN1TR~1GGGQ94W($Hu_26bEy_OZCrItQ^pTpwaK8#n6H*u+qn+ji= z%6apiFu9c_k)splCBu1O{cm`gv?LP0qwY8RqGUAWg=LdugK@V}ajwIU+s`3=sg;X@ z5AsdobgOxD1>3cW{~foXJ?yTJpHZ8_K}@%?eQ_{(o5p zuK!^f&@|a7qa!SXdvQWu4WBL(vcf3U9U|$6vGfF3a3;6N@a70Y-oqSC(|arBHujHN zT>5uY&*!_dUyUl*7D7UQ0Hn3r*BLEFzQBel#ti1MK#l+Cq^Zhp&peSi~& zwNb%S1PBoR%iS=I@pxxVRTIINi0y!&xo=5!kx0Y&ZnCo4gh#BnyWGv>(xQN6!Z8?V zO8^63ckW9JcDOkezwHhdmVaF5C{5g`KE3tfU3th*d4y2@uHX+RARlBz#yp1mM<;~) z_e&wnxGlC(9^0MOgqK~>ZJunm%*9XNYC@zkE^iIRa*N2#t@n&wm)s&%RfOQqFc*EP z_pWtNq9@7vxHyN!Ch_XrEsa*$868;eMw5Dy&v$oBM{CI*ud)frYozg^{_mjlPm;yo z0rkiPU|o+&9H9)`e1n4X7|Ysw+x%`EIZ9n4R9X<7dT+=Ji4ow-yLi(K@*7I@WIj0K zo{gk*zp*Bl)fGO39jb9f{8f`$EGu_PVj6*e?R3)voCwHKqT!GxdZ+iF*Jh&zY}~1c zBvZqD1E}ED-M4ll#Wt~r&t>u8598kd4wJTrb)S{^pLt-ARGkUuze}vM+5^_<^D5_I zE`85$px;R5F_xv*y&K2U6mdhI0xABkNp;{t!WfNIj~&xEs7m0T4npWQcBtQ0R9`?e zH*!6Z6pdF)9K-C_^nVIj=>-Ici4j{R!W50EPclsYeoZn8xhF-0 zKerX!yPgM0;ke>dd{FZ{4j;)?&pVGZFh*8c@fpL~0J4xS?R-O4S>7#eU_6?kW;G}I zia?>48k>g^(b5D~YIq98A|`4TWoHj_wg)BHjb!O*>HD{>1r^Vxoa1 z>{8d*jwR`OXH6kP83T|SMqvqLs69Org6J(0Eb{`2qvOq>s1AsCMf?sLrTHt` zx$@e|z0B!w;T`WUgpXMNtwZ5|>yXs8i%^Nm*!y^|1ULsvRF|u~K_@%9j7m2g&;)OP z08ew%G*aJ+YDBQSgMiIATyaAe9lg?61PA|*ux=F_QAKbWTHd|i3zQ`S2B5`F5a&op?wPFUrp+JeunD}LormCyhE z#YL1!L@^1hEESoDcA?VZ$nw12Uj3wB234*6iMnnfQq1=EP`(?q@<;Aw$NjHArB-ty zWPO7iE^=xp$iF`z6IgD1ecHsE4&;nsQ1e57t88ukeCHr3m~r)e6z%`s?I_j9Hv`D%XFHk^5B4|6 z8g-8Uw?+dJOQ9yNK6G~LU1M`X*4h8O9U?&mffPO@itzbA5~9D42^*LL<4HbE3@bP3 z-@XLL8t54rQ z$XCVy?Y&c#k9wMTO&=LwVmR1gw>FNLP_tc`Mj+e)Q;5HoZe}*8%4b*-t-$OF5 z`=2f$q<&Xef9l!`)%*Pa{jC5=l&N7cvi)^OI(T6F8)3ut=P6V!r-PAdiFT!(%$49X z{GxsA2eP3!pw+02%2m%|$-kn5Onv_N;d7-z20Btkpm{h+eR`edMEvljD{v@iNjt>N98E-zpC>;nMt&dAsa@Sd2cB~@c($PN5{+IO!dEibaeV* zCQOVxH#hGM)g=U}M9)eh@FybtA%i0F z=gfH=vI8$XQ5a_(QfoKR%Gc;gt+7k~?*|n=E36!lgJ3+kU=vZk1FHvXk^kRmbz87i zhhVvOzszbdhwJYVgcO^G4aUnA|5^2SafKqX>)}qjyY;??NYlY;(84Yl82KrK8sL9) z-4k2<{BVIQU{dVmXK&TR!&N1;(hf(EfknEtAS714^+^pt4 zIkkZbSKRG`^?l5Fx{R%O58<|fQ2ZBduQVfClrz9igiA5y3r9saJf978sf*$Bk4PT> z%1$yYD(e%4VH?5=e_MlhkbEA|S4@Djs$GP@@D@#PQ{dH&;%59ZsMq^IgED`nuTSdz zpWk&FD{vc7kRH<+SaOJ{Ed3YP&moiQpyX>M1Z`R(0u>!VNmwLn3U+~lFu!i$`wI9; ztb+%_r~E-M4`Z0Fr_elyT5dGSrqGJk_q3z}=I>1quMd1f02mmJ=pjXHp>{uL}{o>jDrsPV+bFD4#%#S+IK#r8*fLYHC;rMuc=CBJf|lK{^C# zv!Ej_0p$dOnHX@#Lf$YEqL0aI^7Vb{4iuR`f-RI5FJ>;z(ur3PSpb4f93O0~%}*kA zzP_8hWiQlzC^i{12#a&Y))-e{BBAJSIL zyH+<2G*bVt+lJsgKjqxbDQMCXoOHN@3@|CIS-w$aGztL01`JoK1l`p@1vtW-jQ#QX zEB4pr_{G`&NGcC^`5~?qk}Kd=Xzc$t;3A;e|AeoZQB2{Mn&+!pEY-X5CV?=w#b*`Y z^DRp%wVkXAwu(%!HaNaiXemN^X{YG}7p+S@MkY#1yk@odn>^*-;=Up4(HG^j2RYzJ zl}2NBO}+nQMJ%+wXHPMIgHdxK>|cMP`-k_WBO$%;OQF+tCc1T9AM)# zD7UUM1(CW@q!}UsO1fbfMQ{SP=u?HYMXVPKv=^4(4FY!Z-ow=N2Ee}3&q=VW9BVU+ zj$uWuJUnX{K=R#>FYh0)B>saE4G<`iPWw0Gcf2&DT31Wt`}p);Yv>L#NtC*hkt>A) zP?pHUY2zSjeVV;q!HFKOhC>r~KjIiQRS$@KBkD)6eLjrgn5^ zS@Jc^J_87TG$8S1`yn5KM%L&42aOz3G5YR@Zl9af9P1iyq9Ecl9Yp0E97gR66vxR% zkf>PHMw6B1|6mz=%o>mzom2uIWL$FB+@^dl&bTI#{*S;Md$3a)-Ehj~%kyGvElC%@ zb-tCtR_r}XnK~rnd_p3A$?HF-cjJ}c=!85jLK}}l0{%bsMMu&L?BZl^XhV&G8B$!? zBe_5(46lNU;jXa*--{DNckDQdzsSsorMnK}<@r_LJl;5x%Ie~J7B=Aj=jxGvjUH-Z zmO{V{&G|Q(<`58)zhIq-PH?NaOYx?iX+W}kV9#w+87r`To`_**KDmb`$KHkcpX@+8 zF6MFl6cFCK9^9Pf9yWKTG_NxK`i}5kO_!4*Sd) zfdp``bmjL9b+9BBWVQ>{qtNAO7nMXVE6(qm z;hf`{XCBY<_xFC^`@P@y-oE$u(zSTNl6V*B)N4rjVWhPgy$2k4>ZM^hnNizrmzv4q z@_U3%y~@Tlrc6OVilQnfq0PX0>AylMzaNCKw|-pGcQqMI&L``zpOgQF!b!`crM|3N1$P2Oq1JE66lB0-zCXq{2(QM6%s zL9+_ISr-~Rq4zfTLK!O#)k}8jsN~vA89AOyV!O`Xp1Y*~2c)h~V?W}ev4`vDD9k^t z|JZ~eee)-2&J4@gF&#C(i2LT_1mo&oo0*|6ACjBu4WBNSUT~otFNj2})B2ZO-na1Z z#ub$(AAS=!SI2>!otf9XfCH<;OlN(7>(w2tvp+dkl^-J#iK6W@Lo}1vImCU+*_g(tfiLol#@I(S?g|1V-Vk&kr@3sZeKYR`Hf)& zgDVuJCf?fA)GRBd1y=#u5KlENKiGO^sd3=2sTgo*<44ZR5FO2^WxUI`79)%DD&Hb{ zSeZ^Q`F@tLS9x0Yi(gcp6pKWCYkxv`rQPO=Bl1ifDs=i&Wu3(b+|$46k<{3tOCQuA zQ;pfGHxS`dES>L`w`AUAjYz~5mskL2zeTj_zxI95))|RyW3^`~W(wF0o9lNXzYWTn z7($DsKi&GfZv1tIlr0x?t}j{(*!)H<;uJSohDq__M&tgB$|ATK3B=8aNO5vd=c9c3 z>wJ^o2f=}n=qJ_p?9v=B)I<&?s8>u)^jl3o+PdWs4-l{Qo_9^?&A>G9H3s9d>j|!3N9)ssd|rNID+O*y>|F$lSnR|M8+|YN; zUviYogu{GdGgHuVCM>YF-9?UGugs60M_R5#qL+>IIv@|Z3l9fc73PI+X3`Y}#9SJd ztoGbvB*Y|rkL4H`6M$KnVJpF8ipu@q1xiIc5i5>U1Oi{knQ0OMgwJ*P{YOxqeXPlmS(C zDjrxOpOFcAWUiPP0Odr?jZL)e>%`BoV=T+0z~M^w)%D@@93@VMzxaio)p(05O14*a zqV8eA>edLM0YUiGbkU@6O=Ws7HuSS!AsMbW(bV80{lR-OlX?MLSManCsi~SWB%^Dh zEQ`40B_=gb2Yn%{Dgh-I^AwIo0cxLqp9FcO=|h3qS__gXj_`Y7IsnX@K2Hl)|g~80I0IjcKb>g@DsfZ9E8Y_7oExFdQtzQWAsfZDP zf9xY{2_L;QMW)TBIav3!!F~)Mho8T`t91o$6D~e2_@y|A46@LWvK#Gd56MOe^kGx6 zg;I>Wtcf6-_8rsk@H^j_*Qe~dR^V`@dw6)9yt7BZIpV9`o<>$(l?{mc zX8aMa0eE26Q>bCfymet+{|@~07|dQID*Jw$Qpjus>^3p!aN&mc1v)rQ9K`1LAw9cd zzV@ved3$(o$|wM1qxnP)A{LUNr{$f-O@P+nx+0QVB{Z$y z5Y<3ZQ7;yD49+}Z{tP_uUYRW>6Jx?(6JmVYMkZ7vXCia5<=@HS_s|bdegLGFs>q-5 zGhkV*ndfP`np!|I)eCG(=FSoghoj<^b}1mZ8ov}vx*28iJ$n~}DUPM~j@=@QaQWgX zfBF8>RJHS zouRdw5866W_N)}L$7ep;q;GK1^9#2K((&xaodQ*d!3>_07UHzO-S20JWe?e0GTV+3 zMpO?`%=;BuHq6W}iFCt0V*<@{#||L`%Pg+_4AmdV+-g$;b~=#NI-B 0: - logging.info(str(len(instance_ids)) + " instance(s) created") - else: - logging.info("no instance created") - #create waiter to make sure it's running - - logging.info("waiting for instance to become accessible") - waiter = ec2client.get_waiter('instance_status_ok') - waiter.wait( - Filters=[{ - "Name": "instance-status.status", - "Values": ["ok"] - }, { - "Name": "instance-status.reachability", - "Values": ["passed"] - }, { - "Name": "instance-state-name", - "Values": ["running"] - }], - InstanceIds=instance_ids) - - instances_response = ec2client.describe_instances(InstanceIds=instance_ids) - - return instances_response["Reservations"][0]["Instances"] - - -def create_pservers(): - try: - return run_instances( - image_id=args.pserver_image_id, - instance_type=args.pserver_instance_type, - count=args.pserver_count, - role="PSERVER", ) - except Exception: - logging.exception("error while trying to create pservers") - cleanup(args.task_name) - - -def save_metrics_data(str_msg): - #parse msg - logging.info("found metrics data, saving it to csv file") - global is_metrics_file_created - metrics_raw = str_msg.split(",") - with open(args.log_path + metrics_csv_file_name, 'a') as csvfile: - csv_fieldnames = [] - csv_write_data = {} - for metric in metrics_raw: - metric_data = metric.split("=") - metric_key = metric_data[0].strip() - metric_val = float(metric_data[1].strip()) - if not metric_key in metrics: - metrics[metric_key] = [] - metric_repo = metrics[metric_key] - metric_repo.append(metric_val) - csv_fieldnames.append(metric_key) - csv_write_data[metric_key] = metric_val - writer = csv.DictWriter(csvfile, fieldnames=csv_fieldnames) - if not is_metrics_file_created: - writer.writeheader() - is_metrics_file_created = True - writer.writerow(csv_write_data) - logging.info("csv file appended") - - -def log_to_file(source, filename): - if not filename in log_files: - log_files.append(filename) - with open(args.log_path + filename, "a") as log_file: - for line in iter(source.readline, ""): - log_file.write(line) - if (line.startswith(args.metric_data_identifier)): - #found key data, trying to add to csv - line = line.replace(args.metric_data_identifier, "") - save_metrics_data(line) - - -def parse_command(command_raw, defaults={}): - if not command_raw: - command_raw = "" - commands_processed = [] - parameter_map = copy.copy(defaults) - for seg in command_raw.split(","): - if ":" in seg: - parameters = seg.split(":") - parameter_map[parameters[0]] = parameters[1] - else: - commands_processed.append(seg) - for key, val in parameter_map.iteritems(): - commands_processed.append("--" + key + " " + str(val)) - return " ".join(commands_processed) - - -def create_trainers(kickoff_cmd, pserver_endpoints_str): - def create_and_start_trainer(trainer_index): - logging.info("trainer " + str(trainer_index) + " is starting") - - instance_response = run_instances( - image_id=args.trainer_image_id, - instance_type=args.trainer_instance_type, - count=1, - role="TRAINER", )[0] - trainer_ip = instance_response["PrivateIpAddress"] - - logging.info("trainer " + str(trainer_index) + " started") - - ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(hostname=trainer_ip, username="ubuntu", pkey=ssh_key) - - logging.info("trainer " + str(trainer_index) + - " terminal connected via ssh") - - cmd = kickoff_cmd.format( - PSERVER_HOSTS=pserver_endpoints_str, - DOCKER_IMAGE=args.docker_image, - TRAINER_INDEX=str(trainer_index), - TASK_NAME=args.task_name, - TRAINER_COUNT=args.trainer_count, - COMMAND=parse_command(args.trainer_command, {"device": "GPU"}), - MASTER_ENDPOINT=args.master_server_ip + ":" + - str(args.master_server_port)) - logging.info(cmd) - - stdin, stdout, stderr = ssh_client.exec_command(command=cmd) - - # read and save output log - - logging.info("trainer " + str(trainer_index) + - " command executed, keep fetching log") - - stdout_thread = threading.Thread( - target=log_to_file, - args=( - stdout, - "trainer_" + str(trainer_index) + ".log", )) - stderr_thread = threading.Thread( - target=log_to_file, - args=( - stderr, - "trainer_" + str(trainer_index) + "_err.log", )) - stdout_thread.start() - stderr_thread.start() - - stdout_thread.join() - stderr_thread.join() - - return_code = stdout.channel.recv_exit_status() - if return_code != 0: - trainer_create_results[trainer_index] = {'has_error': True} - raise ValueError("trainer didn't finish with exit code 0") - - ssh_client.close() - - # multi thread starting trainer instance and run kickoff command - - trainer_threads = [] - trainer_create_results = {} - try: - for i in xrange(args.trainer_count): - logging.info("starting tread for trainer " + str(i)) - trainer_thread = threading.Thread( - target=create_and_start_trainer, args=(i, )) - trainer_thread.start() - trainer_threads.append(trainer_thread) - - for trainer_thread in trainer_threads: - trainer_thread.join() - - for result in trainer_create_results: - if result["has_error"]: - logging.error( - "error during trainer starting or training, destorying the while cluster " - ) - cleanup(args.task_name) - break - - logging.info("all trainers stopped") - except Exception, e: - logging.info( - "Training exception, clean up resources, please check log for more info" - ) - finally: - cleanup(args.task_name) - - -def cleanup(task_name): - if args.no_clean_up: - logging.info("no clean up option set, going to leave the setup running") - return - #shutdown all ec2 instances - print("going to clean up " + task_name + " instances") - instances_response = ec2client.describe_instances(Filters=[{ - "Name": "tag:Task_name", - "Values": [task_name] - }]) - - instance_ids = [] - if len(instances_response["Reservations"]) > 0: - for reservation in instances_response["Reservations"]: - for instance in reservation["Instances"]: - instance_ids.append(instance["InstanceId"]) - - ec2client.terminate_instances(InstanceIds=instance_ids) - - instance_termination_waiter = ec2client.get_waiter( - 'instance_terminated') - instance_termination_waiter.wait(InstanceIds=instance_ids) - - #delete the subnet created - - subnet = ec2client.describe_subnets(Filters=[{ - "Name": "tag:Task_name", - "Values": [task_name] - }]) - - if len(subnet["Subnets"]) > 0: - ec2client.delete_subnet(SubnetId=subnet["Subnets"][0]["SubnetId"]) - # no subnet delete waiter, just leave it. - logging.info("Clearnup done") - return - - -def kickoff_pserver(host, pserver_endpoints_str): - try: - ssh_key = paramiko.RSAKey.from_private_key_file(args.pem_path) - ssh_client = paramiko.SSHClient() - ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh_client.connect(hostname=host, username="ubuntu", pkey=ssh_key) - cmd = (script_to_str(args.pserver_bash_file)).format( - PSERVER_HOSTS=pserver_endpoints_str, - DOCKER_IMAGE=args.docker_image, - PSERVER_PORT=args.pserver_port, - TASK_NAME=args.task_name, - COMMAND=parse_command(args.pserver_command, {"device": "CPU"}), - TRAINER_COUNT=args.trainer_count, - TRAINER_INDEX=0, - # there is no way to use 0.0.0.0:port to start pserver - # has to docker --network="host" with host ip to make this work - SERVER_ENDPOINT=host + ":" + str(args.pserver_port), - MASTER_ENDPOINT=args.master_server_ip + ":" + - str(args.master_server_port)) - logging.info(cmd) - stdin, stdout, stderr = ssh_client.exec_command(command=cmd) - - stdout_thread = threading.Thread( - target=log_to_file, args=( - stdout, - "pserver_" + host + ".log", )) - stderr_thread = threading.Thread( - target=log_to_file, args=( - stderr, - "pserver_" + host + "_err.log", )) - stdout_thread.start() - stderr_thread.start() - - stdout_thread.join() - stderr_thread.join() - - return_code = stdout.channel.recv_exit_status() - logging.info(return_code) - if return_code != 0: - raise Exception("Error while kicking off pserver training process") - except Exception: - logging.exception("Error while kicking off pserver training process") - cleanup(args.task_name) - finally: - ssh_client.close() - - -def init_args(): - - if not args.task_name: - args.task_name = generate_task_name() - logging.info("task name generated %s" % (args.task_name)) - - if not args.pem_path: - args.pem_path = os.path.expanduser("~") + "/" + args.key_name + ".pem" - if args.security_group_id: - args.security_group_ids = (args.security_group_id, ) - - args.trainers_job_done_count = 0 - - -def create_cluster(): - - if not args.subnet_id: - logging.info("creating subnet for this task") - args.subnet_id = create_subnet() - logging.info("subnet %s created" % (args.subnet_id)) - - logging.info("creating pservers") - pserver_create_response = create_pservers() - logging.info("pserver created, collecting pserver ips") - - pserver_endpoints = [] - for pserver in pserver_create_response: - pserver_endpoints.append(pserver["NetworkInterfaces"][0][ - "PrivateIpAddress"] + ":" + args.pserver_port) - - pserver_endpoints_str = ",".join(pserver_endpoints) - - logging.info("kicking off pserver training process") - pserver_threads = [] - for pserver in pserver_create_response: - pserver_thread = threading.Thread( - target=kickoff_pserver, - args=(pserver["PrivateIpAddress"], pserver_endpoints_str)) - pserver_thread.start() - pserver_threads.append(pserver_thread) - - logging.info("all pserver training process started") - - logging.info("creating trainers and kicking off trainer training process") - create_trainers( - kickoff_cmd=script_to_str(args.trainer_bash_file), - pserver_endpoints_str=pserver_endpoints_str) - - for pserver_thread in pserver_threads: - pserver_thread.join() - - logging.info("all process ended") - - -def start_server(args): - class S(BaseHTTPRequestHandler): - def _set_headers(self): - self.send_response(200) - self.send_header('Content-type', 'text/text') - self.end_headers() - - def do_HEAD(self): - self._set_headers() - - def do_404(self): - self.send_response(404) - self.send_header('Content-type', 'text/text') - self.end_headers() - logging.info("Received invalid GET request" + self.path) - self.wfile.write("NO ACTION FOUND") - - def do_GET(self): - - request_path = self.path - if request_path == "/status" or request_path == "/master_logs": - self._set_headers() - logging.info("Received request to return status") - with open(args.log_path + "master.log", "r") as logfile: - self.wfile.write(logfile.read().strip()) - elif request_path == "/list_logs" or request_path == "/logs": - self._set_headers() - self.wfile.write("\n".join(log_files)) - elif "/log/" in request_path: - self._set_headers() - log_file_path = request_path.replace("/log/", "") - logging.info("requesting log file path is" + args.log_path + - log_file_path) - with open(args.log_path + log_file_path, "r") as logfile: - self.wfile.write(logfile.read().strip()) - else: - self.do_404() - - def do_POST(self): - - request_path = self.path - - if request_path == "/save_data": - self._set_headers() - logging.info("Received request to save data") - self.wfile.write("DATA SAVED!") - content_length = int(self.headers['Content-Length']) - post_data = self.rfile.read(content_length) - if args.task_name: - with open(args.task_name + ".txt", "a") as text_file: - text_file.write(post_data + "\n") - - elif request_path == "/cleanup": - self._set_headers() - logging.info("Received request to cleanup cluster") - args.no_clean_up = False - cleanup(args.task_name) - self.wfile.write("cleanup in progress") - - else: - self.do_404() - - server_address = ('', args.master_server_port) - httpd = HTTPServer(server_address, S) - logging.info("HTTP server is starting") - httpd.serve_forever() - - -def print_arguments(): - logging.info('----------- Configuration Arguments -----------') - for arg, value in sorted(vars(args).iteritems()): - logging.info('%s: %s' % (arg, value)) - logging.info('------------------------------------------------') - - -if __name__ == "__main__": - print_arguments() - if args.action == "create": - logging.info("going to create cluster") - if not args.key_name or not args.security_group_id: - raise ValueError("key_name and security_group_id are required") - init_args() - create_cluster() - elif args.action == "cleanup": - logging.info("going to cleanup cluster") - if not args.task_name: - raise ValueError("task_name is required") - cleanup(args.task_name) - elif args.action == "serve": - # serve mode - if not args.master_server_ip: - raise ValueError( - "No master server ip set, please run with --action create") - - logging.info("going to start serve and create cluster") - - init_args() - - logging.info("starting server in another thread") - server_thread = threading.Thread(target=start_server, args=(args, )) - server_thread.start() - - create_cluster() - server_thread.join() - elif args.action == "test": - start_server(args) diff --git a/tools/aws_benchmarking/server/logs/master.log b/tools/aws_benchmarking/server/logs/master.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/aws_benchmarking/server/pserver.sh.template b/tools/aws_benchmarking/server/pserver.sh.template deleted file mode 100644 index 8d7f9e84c7..0000000000 --- a/tools/aws_benchmarking/server/pserver.sh.template +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker run --network="host" -i -e "SERVER_ENDPOINT={SERVER_ENDPOINT}" -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "TRAINING_ROLE=PSERVER" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file diff --git a/tools/aws_benchmarking/server/requirements.txt b/tools/aws_benchmarking/server/requirements.txt deleted file mode 100644 index 5c523854f2..0000000000 --- a/tools/aws_benchmarking/server/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -netaddr==0.7.19 -boto3==1.6.21 -namesgenerator==0.3 -paramiko==2.4.1 diff --git a/tools/aws_benchmarking/server/trainer.sh.template b/tools/aws_benchmarking/server/trainer.sh.template deleted file mode 100644 index 9b0aae9f7a..0000000000 --- a/tools/aws_benchmarking/server/trainer.sh.template +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -nvidia-docker run --network="host" -i -e "MASTER_ENDPOINT={MASTER_ENDPOINT}" -e "TASK_NAME={TASK_NAME}" -e "TRAINER_COUNT={TRAINER_COUNT}" -e "TRAINERS={TRAINER_COUNT}" -e "TRAINER_INDEX={TRAINER_INDEX}" -e "PADDLE_INIT_TRAINER_ID={TRAINER_INDEX}" -e "TRAINING_ROLE=TRAINER" -e "PSERVER_HOSTS={PSERVER_HOSTS}" -e "PSERVERS={PSERVER_HOSTS}" {DOCKER_IMAGE} {COMMAND} \ No newline at end of file