This commit is contained in:
tomasz 2020-06-30 21:51:10 +02:00
parent e3547abf75
commit ebd423139c
11 changed files with 902 additions and 42 deletions

3
.gitignore vendored
View File

@ -9,4 +9,5 @@ MY_INFO
*.pyc
*.pyx
*.jpg
*.jpg
*.csv

821
cfg/yolov3-spp-19cls.cfg Normal file
View File

@ -0,0 +1,821 @@
[net]
# Testing
# batch=1
# subdivisions=1
# Training
batch=64
subdivisions=16
width=608
height=608
channels=3
momentum=0.9
decay=0.0005
angle=0
saturation = 1.5
exposure = 1.5
hue=.1
learning_rate=0.001
burn_in=1000
max_batches = 500200
policy=steps
steps=400000,450000
scales=.1,.1
[convolutional]
batch_normalize=1
filters=32
size=3
stride=1
pad=1
activation=leaky
# Downsample
[convolutional]
batch_normalize=1
filters=64
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=32
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=128
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=64
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=64
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=256
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=512
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
# Downsample
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=2
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
filters=1024
size=3
stride=1
pad=1
activation=leaky
[shortcut]
from=-3
activation=linear
######################
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
### SPP ###
[maxpool]
stride=1
size=5
[route]
layers=-2
[maxpool]
stride=1
size=9
[route]
layers=-4
[maxpool]
stride=1
size=13
[route]
layers=-1,-3,-5,-6
### End SPP ###
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
batch_normalize=1
filters=512
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=1024
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=72
activation=linear
[yolo]
mask = 6,7,8
anchors=26,16, 19,45, 36,27, 54,32, 34,80, 60,49, 74,71, 96,105, 135,145
classes=19
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1
[route]
layers = -4
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[upsample]
stride=2
[route]
layers = -1, 61
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
batch_normalize=1
filters=256
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=512
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=72
activation=linear
[yolo]
mask = 3,4,5
anchors=26,16, 19,45, 36,27, 54,32, 34,80, 60,49, 74,71, 96,105, 135,145
classes=19
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1
[route]
layers = -4
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[upsample]
stride=2
[route]
layers = -1, 36
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
batch_normalize=1
filters=128
size=1
stride=1
pad=1
activation=leaky
[convolutional]
batch_normalize=1
size=3
stride=1
pad=1
filters=256
activation=leaky
[convolutional]
size=1
stride=1
pad=1
filters=72
activation=linear
[yolo]
mask = 0,1,2
anchors=26,16, 19,45, 36,27, 54,32, 34,80, 60,49, 74,71, 96,105, 135,145
classes=19
num=9
jitter=.3
ignore_thresh = .7
truth_thresh = 1
random=1

6
data/widok_01_19.data Normal file
View File

@ -0,0 +1,6 @@
classes=19
train=/home/tomekb/yolov3/data/widok_01_19/widok_01_19_train_labels.txt
valid=/home/tomekb/yolov3/data/widok_01_19/widok_01_19_test_labels.txt
names=/home/tomekb/yolov3/data/widok_01_19/widok_01_19.names
backup=backup/
eval=coco

View File

@ -126,7 +126,7 @@ def detect(save_img=False):
if save_txt: # Write to file
xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh
with open(save_path[:save_path.rfind('.')] + '.txt', 'a') as file:
file.write(('%g ' * 5 + '\n') % (cls, *xywh)) # label format
file.write(('%g ' * 6 + '\n') % (cls, *xywh, conf)) # label format
if save_img or view_img: # Add bbox to image
label = '%s %.2f' % (names[int(cls)], conf)

View File

@ -1,18 +1,18 @@
train:
epochs: 2
batch-size: 400
cfg: ./cfg/yolov3-spp-18cls.cfg
data: ./data/widok01-11.data
epochs: 600
batch-size: 8
cfg: ./cfg/yolov3-spp-19cls.cfg
data: ./data/widok_01_19.data
multi-scale: false
img-size: '64 128'
img-size: '512 960'
rect: false
resume: false
resume: true
nosave: false
notest: false
evolve: false
bucket:
cache-images: false
weights: /home/tomekb/yolov3/weights/yolov3-spp-ultralytics.pt
weights: /home/tomekb/yolov3/weights/last.pt
device: 1
adam: true
single-cls: false
@ -41,16 +41,15 @@ train:
experiments:
dir: ./experiments
detect:
source: /home/michall/yolov3/data/widok01-11_test_labels.txt
source: /home/tomekb/yolov3/data/widok_01_19/widok_01_19_test_labels.txt
test-img-size: 1024
conf-thres: 0.3
iou-thres: 0.6
save-txt: true
classes:
agnostic-nms:
augment:
confussion-matrix:
labels-dir: ./data/widok01-11_labels
labels-dir: /home/tomekb/yolov3/data/widok_01_19/widok_01_19_labels
bayes:
todo: todo

View File

@ -62,6 +62,8 @@ function findNearest(position, boxes) {
}
function compare(labels, detections) {
// console.log("LABELS", JSON.stringify(labels))
// console.log("DETECTIONS", JSON.stringify(detections))
const results = {}
for(const label of labels) {
const detection = findNearest(label, detections)
@ -88,7 +90,7 @@ async function compareLabelsAndResults(txt) {
const basename = path.basename(txt.split('.')[0])
const labelPath = path.resolve(labelsDir, basename+'.txt')
const [detectionData, labelData] = await Promise.all([fs.promises.readFile(detectionPath, 'utf8'), fs.promises.readFile(labelPath, 'utf8')])
const detections = parseDetections(detectionData)
const detections = parseLabels(detectionData)
const labels = parseLabels(labelData)
return {

View File

@ -46,7 +46,7 @@ def map_class_name_to_id(class_name, xml_document, class_distribution):
elif class_name in ['6. osobowy van 7-9']:
class_distribution[5] += 1
return 5
elif class_name in ['7. dostawczy blaszak']:
elif class_name in ['7. dostawczy blaszak', '27. BUS sredni dostawczy']:
class_distribution[6] += 1
return 6
elif class_name in ['8. dostawczy zabudowany']:
@ -55,54 +55,51 @@ def map_class_name_to_id(class_name, xml_document, class_distribution):
elif class_name in ['9. dostawczy pickup (w tym pomoc drog.)']:
class_distribution[8] += 1
return 8
elif class_name in ['10. dostawczy VAN (osobowy)']:
elif class_name in ['10. dostawczy VAN (osobowy)', '11. autobus maly 10-24', '28. BUS sredni osobowy']:
class_distribution[9] += 1
return 9
elif class_name in ['11. autobus mały 10-24']:
return -1
elif class_name in ['12. autobus miejski']:
elif class_name in ['12. autobus miejski', '13. autobus turystyczny i inny']:
class_distribution[10] += 1
return 10
elif class_name in ['13. autobus turystyczny i inny']:
return -1
elif class_name in ['14. ciężarowy pow. 3,5t zabudowany']:
elif class_name in ['14. ciezarowy pow. 3,5t zabudowany', '17. ciezarowy z widoczna przyczepa']:
class_distribution[11] += 1
return 11
elif class_name in ['15. ciężarowy pow. 3,5t otwarty (w tym duży holownik)']:
elif class_name in ['15. ciezarowy pow. 3,5t otwarty (w tym duzy holownik)']:
class_distribution[12] += 1
return 12
elif class_name in ['16. ciężarowy pow. 3,5t inny (wanna, gruszka, dźwig itp.)']:
elif class_name in ['16. ciezarowy pow. 3,5t inny (wanna, gruszka, dzwig itp.)']:
class_distribution[13] += 1
return 13
elif class_name in ['17. ciężarowy z widoczną przyczepą']:
return -1
elif class_name in ['18. ciągnik siodłowy z widoczną naczepą']:
elif class_name in ['18. ciagnik siodlowy z widoczna naczepa', '19. ciagnik siodlowy bez naczepy']:
class_distribution[14] += 1
return 14
elif class_name in ['19. ciągnik siodłowy bez naczepy']:
elif class_name in ['23. inne pojazdy silnikowe', '20. camper', '21. woz strazacki', '22. ciagnik roliczy, koparka, spychacz']:
class_distribution[15] += 1
return 15
elif class_name in ['20. camper']:
class_distribution[15] += 1
return -1
elif class_name in ['22. ciągnik roliczy, koparka, spychacz']:
return -1
elif class_name in ['23. inne pojazdy silnikowe']:
return -1
elif class_name in ['24. przyczepa']:
class_distribution[16] += 1
return 16
elif class_name in ['25. BUS-karetka/policja']:
class_distribution[17] += 1
return 17
elif class_name in ['26. BUS brygadowka']:
class_distribution[18] += 1
return 18
else:
raise Exception('Unknown Class ', xml_document, class_name)
#print(f'{xml_document.split("/")[-1]} {class_name}')
def generate_txt_from_xml():
class_distribution = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
class_distribution = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
filepaths = glob(join(annotations , '*.xml'))
duplicates = set([x for x in filepaths if filepaths.count(x) > 1])
if duplicates:
print('Duplicate files : ')
print(duplicates)
filepaths = glob(annotations + '*.xml')
for filepath in filepaths:
txtpath = join(modified_annotations, re.sub(r"\.xml$", ".txt", os.path.basename(filepath)))
@ -110,6 +107,14 @@ def generate_txt_from_xml():
tree = ET.parse(in_file)
root = tree.getroot()
print(filepath)
if root.attrib == {}:
continue
if root.attrib['verified'] != 'yes':
continue
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)

View File

@ -46,6 +46,7 @@ def move_training_results_to_experiments_dir(config):
#for test purposes only
#shutil.copy2('/home/tomekb/yolov3/experiments/1/best.pt', training_results_dir_path)
return weights_path, experiment_names_path, training_results_dir_path
@ -60,8 +61,8 @@ def call_detection_script(config, weights_path, names_path, dir):
--weights {weights_path}
--test-img-size {getattr(config.detect, 'test-img-size')}
--conf-thres {getattr(config.detect, 'conf-thres')}
--iou-thres {getattr(config.detect, 'iou-thres')}"""
cmd += " --save-txt" if getattr(config.detect, 'save-txt') else ""
--iou-thres {getattr(config.detect, 'iou-thres')}
--save-txt"""
cmd += " --agnostic-nms" if getattr(config.detect, 'agnostic-nms') else ""
cmd += " --agument" if getattr(config.detect, 'augment') else ""
cmd += f" --device {config.train.device}" if config.train.device else ""
@ -75,7 +76,7 @@ def call_detection_script(config, weights_path, names_path, dir):
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,shell=True)
for line in io.TextIOWrapper(process.stdout, encoding="utf-8"): # print output of process to console
print(line)
return detect_output_dir

View File

@ -0,0 +1,24 @@
import os
import sys
import csv
# This should just be a file which groups files according to train, val datasets
csv_file = sys.argv[1]
jpg_files_dir = sys.argv[2]
output_train_labels = sys.argv[3]
output_test_labels = sys.argv[4]
with open(csv_file, "r") as f:
reader = csv.reader(f, delimiter=",")
train_labels = open(output_train_labels, 'w')
test_labels = open(output_test_labels, 'w')
for i, line in enumerate(reader):
if i != 0: # skip header
file, train_or_test = line
full_file_path = os.path.join(jpg_files_dir, file + '.jpg')
to_write = [full_file_path, '\n']
if train_or_test == 'train':
train_labels.writelines(to_write)
else:
test_labels.writelines(to_write)

View File

@ -117,6 +117,7 @@ def train(hyp):
best_fitness = 0.0
attempt_download(weights)
if weights.endswith('.pt'): # pytorch format
print("LOADIN MODEL")
# possible weights are '*.pt', 'yolov3-spp.pt', 'yolov3-tiny.pt' etc.
chkpt = torch.load(weights, map_location=device)
@ -397,7 +398,7 @@ if __name__ == '__main__':
parser.add_argument('--adam', action='store_true', help='use adam optimizer')
parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
opt = parser.parse_args()
opt.weights = last if opt.resume else opt.weights
#opt.weights = last if opt.resume else opt.weights
check_git_status()
opt.cfg = check_file(opt.cfg) # check file
opt.data = check_file(opt.data) # check file

View File

@ -830,10 +830,10 @@ def plot_one_box(x, img, color=None, label=None, line_thickness=None):
cv2.rectangle(img, c1, c2, color, thickness=tl, lineType=cv2.LINE_AA)
if label:
tf = max(tl - 1, 1) # font thickness
t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]
t_size = cv2.getTextSize(label, 0, fontScale=tl / 5, thickness=tf)[0]
c2 = c1[0] + t_size[0], c1[1] - t_size[1] - 3
cv2.rectangle(img, c1, c2, color, -1, cv2.LINE_AA) # filled
cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 3, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)
cv2.putText(img, label, (c1[0], c1[1] - 2), 0, tl / 5, [225, 255, 255], thickness=tf, lineType=cv2.LINE_AA)
def plot_wh_methods(): # from utils.utils import *; plot_wh_methods()