car-detection-bayes/utils/gcp.sh

136 lines
8.3 KiB
Bash
Raw Normal View History

2018-08-26 08:51:39 +00:00
#!/usr/bin/env bash
2019-02-20 22:21:42 +00:00
# New VM
2019-08-04 18:03:46 +00:00
rm -rf sample_data yolov3 darknet apex coco cocoapi knife knifec
2019-03-25 17:35:39 +00:00
git clone https://github.com/ultralytics/yolov3
# git clone https://github.com/AlexeyAB/darknet && cd darknet && make GPU=1 CUDNN=1 CUDNN_HALF=1 OPENCV=0 && wget -c https://pjreddie.com/media/files/darknet53.conv.74 && cd ..
git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . --user && cd .. && rm -rf apex
2019-11-12 22:44:45 +00:00
sudo conda install -y -c conda-forge scikit-image pycocotools # tensorboard
2019-08-04 18:03:46 +00:00
python3 -c "
from yolov3.utils.google_utils import gdrive_download
2019-08-31 11:21:15 +00:00
gdrive_download('1HaXkef9z6y5l4vUnCYgdmEAj61c6bfWO','coco.zip')"
2019-04-24 15:02:52 +00:00
sudo shutdown
2019-02-20 22:21:42 +00:00
2019-03-25 18:39:14 +00:00
# Re-clone
2019-05-13 19:18:15 +00:00
rm -rf yolov3 # Warning: remove existing
2019-08-24 14:43:43 +00:00
git clone https://github.com/ultralytics/yolov3 && cd yolov3 # master
2019-04-24 12:19:43 +00:00
# git clone -b test --depth 1 https://github.com/ultralytics/yolov3 test # branch
2019-08-26 12:47:36 +00:00
python3 train.py --img-size 320 --weights weights/darknet53.conv.74 --epochs 27 --batch-size 64 --accumulate 1
2019-03-25 18:39:14 +00:00
# Train
python3 train.py
2018-08-26 08:51:39 +00:00
# Resume
python3 train.py --resume
2018-08-26 08:51:39 +00:00
# Detect
2019-03-22 15:53:06 +00:00
python3 detect.py
2018-08-26 08:51:39 +00:00
2018-09-04 12:36:51 +00:00
# Test
python3 test.py --save-json
2018-09-10 14:41:02 +00:00
2019-07-24 18:16:35 +00:00
# Evolve
2019-07-24 18:32:24 +00:00
for i in {0..500}
2019-07-24 18:16:35 +00:00
do
2019-11-26 20:26:14 +00:00
python3 train.py --data data/coco.data --img-size 416 --epochs 27 --batch-size 32 --accumulate 2 --evolve --weights '' --bucket yolov4/416_coco_27e --device 0
2019-07-24 18:16:35 +00:00
done
2019-03-25 18:39:14 +00:00
# Git pull
git pull https://github.com/ultralytics/yolov3 # master
2019-04-06 14:16:40 +00:00
git pull https://github.com/ultralytics/yolov3 test # branch
2018-12-04 18:17:03 +00:00
2019-03-25 18:39:14 +00:00
# Test Darknet training
python3 test.py --weights ../darknet/backup/yolov3.backup
2018-10-09 17:22:33 +00:00
2019-07-15 15:54:31 +00:00
# Copy last.pt TO bucket
gsutil cp yolov3/weights/last1gpu.pt gs://ultralytics
2018-11-13 11:20:01 +00:00
2019-07-15 15:54:31 +00:00
# Copy last.pt FROM bucket
gsutil cp gs://ultralytics/last.pt yolov3/weights/last.pt
wget https://storage.googleapis.com/ultralytics/yolov3/last_v1_0.pt -O weights/last_v1_0.pt
wget https://storage.googleapis.com/ultralytics/yolov3/best_v1_0.pt -O weights/best_v1_0.pt
2018-11-13 11:20:01 +00:00
2019-04-18 14:07:47 +00:00
# Reproduce tutorials
2019-04-16 10:49:34 +00:00
rm results*.txt # WARNING: removes existing results
2019-05-08 11:31:49 +00:00
python3 train.py --nosave --data data/coco_1img.data && mv results.txt results0r_1img.txt
python3 train.py --nosave --data data/coco_10img.data && mv results.txt results0r_10img.txt
python3 train.py --nosave --data data/coco_100img.data && mv results.txt results0r_100img.txt
2019-07-12 13:44:39 +00:00
# python3 train.py --nosave --data data/coco_100img.data --transfer && mv results.txt results3_100imgTL.txt
2019-04-16 10:49:34 +00:00
python3 -c "from utils import utils; utils.plot_results()"
2019-07-12 13:44:39 +00:00
# gsutil cp results*.txt gs://ultralytics
2019-04-16 10:49:34 +00:00
gsutil cp results.png gs://ultralytics
sudo shutdown
2019-04-02 16:04:04 +00:00
2019-04-23 16:53:36 +00:00
# Reproduce mAP
2019-05-02 22:26:26 +00:00
python3 test.py --save-json --img-size 608
2019-04-23 16:53:36 +00:00
python3 test.py --save-json --img-size 416
python3 test.py --save-json --img-size 320
sudo shutdown
2019-08-05 11:50:06 +00:00
# Benchmark script
git clone https://github.com/ultralytics/yolov3 # clone our repo
git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . --user && cd .. && rm -rf apex # install nvidia apex
python3 -c "from yolov3.utils.google_utils import gdrive_download; gdrive_download('1HaXkef9z6y5l4vUnCYgdmEAj61c6bfWO','coco.zip')" # download coco dataset (20GB)
2019-08-05 12:15:12 +00:00
cd yolov3 && clear && python3 train.py --epochs 1 # run benchmark (~30 min)
2019-08-05 11:50:06 +00:00
2019-04-19 11:18:47 +00:00
# Unit tests
2019-04-24 19:41:18 +00:00
python3 detect.py # detect 2 persons, 1 tie
2019-05-18 21:24:26 +00:00
python3 test.py --data data/coco_32img.data # test mAP = 0.8
python3 train.py --data data/coco_32img.data --epochs 5 --nosave # train 5 epochs
python3 train.py --data data/coco_1cls.data --epochs 5 --nosave # train 5 epochs
python3 train.py --data data/coco_1img.data --epochs 5 --nosave # train 5 epochs
2019-04-19 11:18:47 +00:00
2019-05-13 19:18:15 +00:00
# AlexyAB Darknet
2019-06-12 11:19:17 +00:00
gsutil cp -r gs://sm6/supermarket2 . # dataset from bucket
2019-05-21 13:00:16 +00:00
rm -rf darknet && git clone https://github.com/AlexeyAB/darknet && cd darknet && wget -c https://pjreddie.com/media/files/darknet53.conv.74 # sudo apt install libopencv-dev && make
2019-05-29 23:40:35 +00:00
./darknet detector calc_anchors data/coco_img64.data -num_of_clusters 9 -width 320 -height 320 # kmeans anchor calculation
2019-06-12 11:19:17 +00:00
./darknet detector train ../supermarket2/supermarket2.data ../yolo_v3_spp_pan_scale.cfg darknet53.conv.74 -map -dont_show # train spp
2019-05-29 23:40:35 +00:00
./darknet detector train ../yolov3/data/coco.data ../yolov3-spp.cfg darknet53.conv.74 -map -dont_show # train spp coco
2019-09-16 21:10:21 +00:00
#Docker
sudo docker kill $(sudo docker ps -q)
2019-11-15 01:48:06 +00:00
sudo docker pull ultralytics/yolov3:v0
2019-11-26 20:26:14 +00:00
sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco ultralytics/yolov3:v0
2019-09-16 21:10:21 +00:00
clear
while true
do
2019-11-15 01:48:06 +00:00
python3 train.py --weights '' --prebias --img-size 512 --batch-size 32 --accumulate 2 --evolve --epochs 27 --bucket yolov4/512_coco_27e --device 0
2019-09-16 21:10:21 +00:00
done
2019-11-15 01:48:06 +00:00
2019-11-26 20:26:14 +00:00
export tag=ultralytics/yolov3:v70 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 273 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 70 --device 0 --multi
export tag=ultralytics/yolov3:v0 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 273 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 71 --device 0 --multi --img-weights
export tag=ultralytics/yolov3:v73 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 73 --device 5 --cfg cfg/yolov3s.cfg
export tag=ultralytics/yolov3:v74 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 74 --device 0 --cfg cfg/yolov3s.cfg
export tag=ultralytics/yolov3:v75 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 75 --device 7 --cfg cfg/yolov3s.cfg
export tag=ultralytics/yolov3:v76 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 76 --device 0 --cfg cfg/yolov3s.cfg
export tag=ultralytics/yolov3:v79 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 79 --device 5
export tag=ultralytics/yolov3:v80 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 80 --device 0
export tag=ultralytics/yolov3:v81 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 81 --device 7
export tag=ultralytics/yolov3:v82 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag python3 train.py --weights '' --epochs 27 --batch-size 16 --accumulate 4 --prebias --bucket yolov4 --name 82 --device 0 --cfg cfg/yolov3s.cfg
#SM4
export tag=ultralytics/yolov3:v0 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/data,target=/usr/src/data $tag python3 train.py --weights 'ultralytics49.pt' --epochs 500 --img-size 320 --batch-size 32 --accumulate 2 --prebias --bucket yolov4 --name 78 --device 0 --multi --cfg cfg/yolov3-spp-3cls.cfg --data ../data/sm4/out.data
2019-11-15 01:48:06 +00:00
2019-11-26 20:26:14 +00:00
export tag=ultralytics/yolov3:v2 && sudo docker pull $tag && sudo nvidia-docker run -it --ipc=host --mount type=bind,source="$(pwd)"/coco,target=/usr/src/coco $tag
clear
sleep 120
while true
do
python3 train.py --weights '' --epochs 27 --batch-size 32 --accumulate 2 --prebias --evolve --device 7 --bucket yolov4/416_coco_27e
done
2019-11-15 01:48:06 +00:00
2019-11-18 02:48:50 +00:00
while true; do python3 train.py --data data/coco.data --img-size 320 --batch-size 64 --accumulate 1 --evolve --epochs 1 --adam --bucket yolov4/adamdefaultpw_coco_1e; done
2019-11-15 01:48:06 +00:00