car-detection-bayes/our_scripts/run_yolov3_process_bayes.py

184 lines
7.3 KiB
Python
Raw Normal View History

2020-07-25 22:42:33 +00:00
import datetime
import glob
import ntpath
import os
import shutil
2020-07-25 22:43:07 +00:00
import traceback
2020-07-25 22:42:33 +00:00
2020-07-25 22:43:07 +00:00
import GPyOpt
from config_bayes import Configuration
2020-07-26 08:49:20 +00:00
2020-07-29 16:46:58 +00:00
from utils import call_subprocess, get_values_from_conff_matrix, load_previous_bayes_experiments
2020-07-25 22:42:33 +00:00
2020-07-26 08:49:20 +00:00
dir_path = os.path.dirname(os.path.realpath(__file__))
PROJECT_ROOT = os.path.join(dir_path, '..')
bayes_config_yaml = os.path.join(dir_path, 'config_bayes.yml')
config = Configuration(bayes_config_yaml)
2020-07-25 22:43:07 +00:00
date_string = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
2020-07-26 08:49:20 +00:00
bayes_params_file = open(os.path.join(PROJECT_ROOT, config.experiments.dir, f"{date_string}_bayes_params.txt"), 'a+')
2020-07-25 22:43:07 +00:00
def call_training_script(gaussian_hyps):
2020-07-26 08:49:20 +00:00
cmd = 'python -u train.py'
2020-07-25 22:43:07 +00:00
cmd += ' --epochs ' + gaussian_hyps['epochs'].__str__()
cmd += ' --batch-size ' + gaussian_hyps['batch-size'].__str__()
cmd += ' --cfg ' + config.train.cfg.__str__()
cmd += ' --data ' + config.train.data.__str__()
cmd += ' --multi-scale ' if gaussian_hyps['multi-scale'] else ""
cmd += ' --img-size ' + gaussian_hyps['img-size']
cmd += ' --rect ' if gaussian_hyps['rect'] else ""
cmd += ' --weights ' + config.train.weights.__str__()
cmd += ' --device ' + config.train.device.__str__()
cmd += ' --adam ' if gaussian_hyps['adam'] else ""
cmd += ' --freeze-layers ' if getattr(config.train, "freeze-layers") else ""
# cmd += ' --snapshot-every ' if getattr(config.train, "snapshot-every") else ""
cmd += ' --experiment-dir ' + config.experiments.dir.__str__()
train_hyps = dict(
(key, gaussian_hyps[key]) for idx, (key, _) in enumerate(gaussian_hyps.items()) if idx in range(6, 24))
cmd += f' --hyp \"{train_hyps}\"'
2020-07-25 22:42:33 +00:00
print("_______ CALLING TRAINING SCRIPT _______")
print(cmd)
2020-07-25 22:43:07 +00:00
call_subprocess(cmd)
2020-07-25 22:42:33 +00:00
return cmd
2020-07-25 22:43:07 +00:00
def move_training_results_to_experiments_dir():
training_results_dir_path = os.path.join(config.experiments.dir, datetime.datetime.now().strftime(
'%Y-%m-%d_%H-%M-%S')) # creating directory accordint to pattern eg: 2020-06-30_17-52-19
2020-07-25 22:42:33 +00:00
print("_______ CALLING MOVING RESULTS _______")
print(f"MOVING RESUTLS TO {training_results_dir_path}")
os.mkdir(training_results_dir_path)
weights_path = os.path.join(training_results_dir_path, 'best.pt')
2020-07-26 08:49:20 +00:00
shutil.move('./weights/best.pt', weights_path) # move best weights
2020-07-25 22:42:33 +00:00
names_path = open(config.train.data).readlines()[3].split('=')[-1].rstrip() # read names path from file
names_file_name = ntpath.basename(names_path)
experiment_names_path = os.path.join(training_results_dir_path, names_file_name)
2020-07-25 22:43:07 +00:00
shutil.copy(names_path,
experiment_names_path) # copy names file from *.data file to created experiment dir with training results
2020-07-25 22:42:33 +00:00
2020-07-26 08:49:20 +00:00
tensorboard_dir = './runs'
2020-07-25 22:43:07 +00:00
tensorboard_events_files = glob.glob(os.path.join(tensorboard_dir, '*'))
last_modified_events_file = max(tensorboard_events_files, key=os.path.getmtime)
shutil.move(last_modified_events_file,
os.path.join(training_results_dir_path)) # saving related tensorboard dir
2020-07-25 22:42:33 +00:00
shutil.copy2(config.config_path, training_results_dir_path) # copying configuration yaml
return weights_path, experiment_names_path, training_results_dir_path
2020-07-25 22:43:07 +00:00
def call_detection_script(gaussian_hyps, weights_path, names_path, dir):
2020-07-25 22:42:33 +00:00
detect_output_dir = os.path.join(dir, 'output')
2020-07-26 08:49:20 +00:00
cmd = f"""python -u ./detect.py
2020-07-25 22:42:33 +00:00
--cfg {config.train.cfg}
--source {config.detect.source}
--output {detect_output_dir}
--names {names_path}
--weights {weights_path}
2020-09-11 16:52:00 +00:00
--test-img-size {getattr(config.detect, 'test-img-size')}
--conf-thres {getattr(config.detect, 'conf-thres')}
--iou-thres {getattr(config.detect, 'iou-thres')}
2020-07-25 22:42:33 +00:00
--save-txt"""
cmd += f" --device {config.train.device}" if config.train.device else ""
cmd = " ".join(cmd.split())
print("_______ CALLING DETECTION SCRIPT _______")
print(cmd)
2020-07-25 22:43:07 +00:00
call_subprocess(cmd)
2020-07-25 22:42:33 +00:00
return detect_output_dir
2020-07-25 22:43:07 +00:00
def call_generate_confussion_matrix(detect_output_dir, names_path, train_results_dir):
2020-07-25 22:42:33 +00:00
labels_dir = getattr(config.confussion_matrix, 'labels-dir')
2020-07-25 22:43:07 +00:00
conff_matrix_path = os.path.join(train_results_dir, 'confussion-matrix.tsv')
2020-07-26 08:49:20 +00:00
cmd = f"node ./our_scripts/generate-confusion-matrix.js {detect_output_dir} {labels_dir} {names_path} > {conff_matrix_path}"
2020-07-25 22:42:33 +00:00
print("_______ CALLING CONFUSSION MATRIX SCRIPT _______")
print(cmd)
2020-07-25 22:43:07 +00:00
call_subprocess(cmd)
return conff_matrix_path
def yolov3(x):
bayes_hyps = {
'epochs': int(x[:, 0]),
'batch-size': int(x[:, 1]),
'multi-scale': bool(x[:, 2]),
'img-size': f"{int(x[:, 3])} {int(x[:, 4])}",
'rect': bool(x[:, 5]),
'adam': bool(x[:, 6]),
'giou': float(x[:, 7]), # train hyps start index
'cls': float(x[:, 8]),
'cls_pw': float(x[:, 9]),
'obj': float(x[:, 10]),
'obj_pw': float(x[:, 11]),
'iou_t': float(x[:, 12]),
'lr0': float(x[:, 13]),
'lrf': float(x[:, 14]),
'momentum': float(x[:, 15]),
'weight_decay': float(x[:, 16]),
'fl_gamma': float(x[:, 17]),
'hsv_h': float(x[:, 18]),
'hsv_s': float(x[:, 19]),
'hsv_v': float(x[:, 20]),
'degrees': float(x[:, 21]),
'translate': float(x[:, 22]),
'scale': float(x[:, 23]),
'shear': float(x[:, 24]), # train hyps end index
}
try:
call_training_script(bayes_hyps)
weights_path, names_path, train_results_dir = move_training_results_to_experiments_dir()
detect_output_dir = call_detection_script(bayes_hyps, weights_path, names_path, train_results_dir)
conf_matrix_path = call_generate_confussion_matrix(detect_output_dir, names_path, train_results_dir)
2020-09-11 16:52:00 +00:00
y_dict = get_values_from_conff_matrix(conf_matrix_path)
2020-07-25 22:43:07 +00:00
2020-09-11 16:52:00 +00:00
# tutaj wzór na wyliczanie funkcji
y_val = (1 - (y_dict['right'] * 10 - y_dict['false positives'] * 3 - y_dict['mistakes']) / y_dict['labeled']) / 30
2020-07-25 22:43:07 +00:00
# zapisywanie do pliku zadeklarowanego globalnie
line = "\t".join([bayes_hyps.__str__(), str(y_val)])
2020-09-11 16:52:00 +00:00
print('###### line ########')
print(line)
2020-07-25 22:43:07 +00:00
bayes_params_file.writelines([line, '\n'])
return y_val
except:
tb = traceback.format_exc()
2020-09-11 16:52:00 +00:00
y_max_val = 1
2020-07-25 22:43:07 +00:00
print("An error occured during running training-detect-confussion process \n", tb)
2020-09-11 16:52:00 +00:00
print(f"Returning {y_max_val} from current bayessian iteration")
line = "\t".join([bayes_hyps.__str__(), str(y_max_val)])
2020-07-25 22:43:07 +00:00
bayes_params_file.writelines([line, '\n'])
2020-09-11 16:52:00 +00:00
return y_max_val
# na jakiej rozdzielczości jest puszczana detekcja
2020-07-25 22:43:07 +00:00
if __name__ == '__main__':
bounds = config.get_bayes_bounds()
2020-07-25 22:42:33 +00:00
2020-07-26 08:49:20 +00:00
os.chdir(PROJECT_ROOT) # change to project root directory
2020-07-29 16:46:58 +00:00
# wczytywanie z poprzednich eksperymentów plik bayes_params
X, Y = load_previous_bayes_experiments(config.experiments.dir)
2020-07-26 08:49:20 +00:00
2020-09-11 16:52:00 +00:00
constraints = [
{
'name':'img_size_constraint',
'constraint': '(x[:,3] - x[:,4])' # img-size-start - img-size-end <= 0
}
]
2020-07-25 22:43:07 +00:00
bayes_optimizer = GPyOpt.methods.BayesianOptimization(f=yolov3, domain=bounds, X=X, Y=Y, verbosity=True,
2020-09-11 16:52:00 +00:00
initial_design_numdata=5, constraints=constraints, )
2020-07-25 22:43:07 +00:00
bayes_optimizer.run_optimization(config.bayes.iterations, verbosity=True)
bayes_params_file.close()