Commit 9d90fd0c authored by Radim Tylecek's avatar Radim Tylecek

evaluation code added

parent e626e219
## Allow to flag private files
~*
*.bag
*.avi
*.zip
classdef YAML
%YAML Serialize a matlab variable to yaml format
%
% [ X ] = YAML.load( S )
% [ S ] = YAML.dump( X )
%
% [ X ] = YAML.read( filepath )
% YAML.write( filepath, X )
%
% YAML.LOAD takes YAML string S and returns matlab variable X.
% YAML.DUMP takes matlab variable X and converts to YAML string S.
% YAML.READ and YAML.WRITE are convenient methods to load and dump
% YAML format directly from a file.
%
% Examples:
% To serialize matlab object
%
% >> X = struct('matrix', rand(3,4), 'char', 'hello');
% >> S = YAML.dump(X);
% >> disp(S);
% matrix:
% - [0.9571669482429456, 0.14188633862721534]
% - [0.4853756487228412, 0.421761282626275]
% - [0.8002804688888001, 0.9157355251890671]
% char: hello
%
% To decode yaml string
%
% >> X = YAML.load(S);
% >> disp(X)
% matrix: [3x2 double]
% char: 'hello'
%
% See also: xmlread xmlwrite
properties (Constant)
JARFILE = YAML.jarfile
end
methods (Static)
function [ S ] = jarfile()
%JARFILE path to the SnakeYAML jar file
S = fileparts(mfilename('fullpath'));
S = [S filesep 'java' filesep 'snakeyaml-1.9.jar'];
end
function [ X ] = load( S )
%LOAD load matlab object from yaml string
javaaddpath(YAML.JARFILE);
% Load yaml into java obj
yaml = org.yaml.snakeyaml.Yaml;
java_obj = yaml.load(S);
% Convert to matlab object
X = YAML.load_data(java_obj);
end
function [ S ] = dump( X )
%DUMP serialize matlab object into yaml string
javaaddpath(YAML.JARFILE);
% Convert matlab obj to java obj
yaml = org.yaml.snakeyaml.Yaml();
java_obj = YAML.dump_data(X);
% Dump into yaml string
S = char(yaml.dump(java_obj));
end
function [ X ] = read( filepath )
%READ read and decode yaml data from file
fid = fopen(filepath,'r');
S = fscanf(fid,'%c',inf);
fclose(fid);
X = YAML.load( S );
end
function [] = write( filepath, X )
%WRITE serialize and write yaml data to file
S = YAML.dump( X );
fid = fopen(filepath,'w');
fprintf(fid,'%s',S);
fclose(fid);
end
end
methods(Static, Access=private)
function result = load_data( r )
%LOAD_DATA recursively convert java objects
if isa(r, 'char')
result = char(r);
elseif isa(r, 'double')
result = double(r);
elseif isa(r, 'java.util.Date')
result = DateTime(r);
elseif isa(r, 'java.util.List')
result = cell(r.size(),1);
itr = r.iterator();
i = 1;
while itr.hasNext()
result{i} = YAML.load_data(itr.next());
i = i + 1;
end
result = YAML.merge_cell(result);
elseif isa(r, 'java.util.Map')
result = struct;
itr = r.keySet().iterator();
while itr.hasNext()
key = itr.next();
result.(char(key)) = YAML.load_data(...
r.get(java.lang.String(key)));
end
else
error('YAML:load_data:typeError',...
['Unknown data type: ' class(r)]);
end
end
function result = merge_cell( r )
%MERGE_CELL convert cell array to native matrix
% Check eligibility
merge = false;
if all(cellfun(@isnumeric,r))
merge = true;
elseif all(cellfun(@isstruct,r))
f = cellfun(@fieldnames,r,'UniformOutput',false);
if isempty(f) || (numel(unique(cellfun(@numel,f)))==1 && ...
all(cellfun(@(x) all(strcmp(f{1},x)),f)))
merge = true;
end
end
% Merge if scalar or row vector
result = r;
if merge
if all(cellfun(@isscalar,r))
result = [r{:}];
elseif all(cellfun(@isrow,r)) &&...
length(unique(cellfun(@length,r)))==1
result = cat(1,r{:});
end
end
end
function result = dump_data( r )
%DUMP_DATA convert
if ischar(r)
result = java.lang.String(r);
elseif ~isscalar(r)
result = java.util.ArrayList();
if size(r,1)==1
for i = 1:numel(r)
if iscell(r)
result.add(YAML.dump_data(r{i}));
else
result.add(YAML.dump_data(r(i)));
end
end
else
for i = 1:size(r,1)
result.add(YAML.dump_data(r(i,:)));
end
end
elseif isnumeric(r)
result = java.lang.Double(r);
elseif isstruct(r)
result = java.util.LinkedHashMap();
keys = fields(r);
for i = 1:length(keys)
result.put(keys{i},YAML.dump_data(r.(keys{i})));
end
elseif iscell(r)
result = java.util.ArrayList();
result.add(YAML.dump_data(r{1}));
elseif isa(r,'DateTime')
result = java.util.Date(datestr(r));
else
error('YAML:load_data:typeError',...
['Unsupported data type: ' class(r)]);
end
end
end
end
<?xml version="1.0" encoding="UTF-8"?>
<CloudCompare>
<ColorScale version="1">
<Properties>
<name>KBGYRW</name>
<uuid>{5a2c0e52-5ad4-4cc7-9be2-815f55ecb971}</uuid>
<absolute>1</absolute>
<minValue>0</minValue>
<range>1</range>
</Properties>
<Data>
<step r="0" g="0" b="0" pos="0"/>
<step r="0" g="0" b="255" pos="0.05"/>
<step r="0" g="255" b="0" pos="0.1"/>
<step r="255" g="255" b="0" pos="0.15"/>
<step r="255" g="0" b="255" pos="0.2"/>
<step r="255" g="0" b="0" pos="0.25"/>
<step r="255" g="0" b="0" pos="1"/>
</Data>
</ColorScale>
</CloudCompare>
#!/bin/bash
CC="cloudcompare.CloudCompare"
GT="gt/model_10mm_real.ply"
echo "*** Computing accuracy ***"
$CC -silent -auto_save OFF -C_EXPORT_FMT PLY \
-o $1 \
-crop -9:-9:-2:10:7:4 \
-SS SPATIAL 0.01 \
-o $GT \
-REMOVE_ALL_SFS \
-c2c_dist \
-pop_clouds \
-SF_COLOR_SCALE cmap.xml \
-SF_CONVERT_TO_RGB false \
-SS SPATIAL 0.01 \
-save_clouds file "${1%.*}-acc.ply"
echo "*** Computing completeness ***"
$CC -silent -auto_save OFF -C_EXPORT_FMT PLY \
-o $GT \
-o $1 \
-crop -9:-9:-2:10:7:4 \
-REMOVE_ALL_SFS \
-c2c_dist \
-pop_clouds \
-SF_COLOR_SCALE cmap.xml \
-SF_CONVERT_TO_RGB false \
-save_clouds file "${1%.*}-comp.ply"
\ No newline at end of file
function stats = eval_recon(subName, scene, accRatio, compThreshold, histBins, maxRange)
%% EVAL_RECON - evaluate accuracy and completeness of a model
% acc is distance d (in m) such that [accRatio]% of the reconstruction is within d of the ground truth mesh
% comp is the percent of points on the GTM that are within [compThreshold] mm of the reconstruction
filePath = sprintf('../results/%s/%s_%s',subName,subName,scene); % distances to GT
if ~exist([filePath '-acc.ply'],'file')
%% compute distances using CloudCompare 2.10+ - https://www.cloudcompare.org/
% linux: install using "sudo snap install cloudcompare --edge --classic"
if strcmp(scene,'real')
eval(sprintf('!./eval_real_pcl.sh %s.ply %s',filePath,scene));
else
eval(sprintf('!./eval_geom_pcl.sh %s.ply %s',filePath,scene));
end
end
stats.accRatio = accRatio; %0.9;
stats.compThreshold = compThreshold; %0.1;
% histBins = 10;
%% read distances from CloudCompare pcls
compPcl = ply_read([filePath '-comp.ply']);
compDist = abs(compPcl.vertex.('scalar_C2C_absolute_distances'));
accPcl = ply_read([filePath '-acc.ply']);
accDist = abs(accPcl.vertex.('scalar_C2C_absolute_distances'));
%% compute stats
stats.acc = quantile(accDist,accRatio);
stats.comp = mean(compDist<compThreshold);
stats.histBins = linspace(maxRange/histBins/2, maxRange-maxRange/histBins/2, histBins);
stats.histBins = maxRange*logspace(-2,0,histBins);
histComp = hist(compDist,stats.histBins);
stats.histComp = histComp/sum(histComp);
histAcc = hist(accDist,stats.histBins);
stats.histAcc = histAcc/sum(histAcc);
%% plot
figure('Name',subName);
subplot(2,2,1);
bar(stats.histBins,cumsum(stats.histComp)); title(sprintf('completeness: %.1f %%',stats.comp*100)); xlabel('m (GT to X)'); ylim([0 1]);
hold on; plot(compThreshold*[1 1],[0 1],'r-');
subplot(2,2,2);
bar(stats.histBins,cumsum(stats.histAcc)); title(sprintf('accuracy: %.3f m',stats.acc)); xlabel('m (X to GT)'); ylim([0 1]);
hold on; plot([0 maxRange],accRatio*[1 1],'r-');
try
subplot(2,2,3); imshow(imread([filePath '-comp.png']));
subplot(2,2,4); imshow(imread([filePath '-acc.png']));
catch
warning('eval: missing pngs');
end
print([filePath '-geom.pdf'],'-dpdf','-bestfit');
%% evaluate semantics
addpath('render','toolbox');
resDir = '../results';
submissions = {'snbasic-frNone'}; % each has separate folder
subnames = {'SegNet-basic'};
%%
if 0
resName = 'real';
scenes = {'test_around_garden' };
dataPath = '../gt';
evalFrames = 140:10:1480;
evalCams = [0 2];
camType = 'uvc_camera_cam';
else
resName = 'test';
scenes = { 'clear_0288','cloudy_0288','overcast_0288','sunset_0288','twilight_0288'};
dataPath = '../gt';
evalFrames = 1:100;
evalCams = 0:2:9;
camType = 'vcam';
end
%% read label def
def = read_labels('../calibration');
labelCount = length(def.labelNames);
%% subs
for m = 1:length(submissions)
subName = submissions{m};
subTitle = subnames{m};
mstats.conf = zeros(labelCount-1,labelCount-1,length(scenes));
mstats.acc = zeros(length(scenes),length(evalCams));
%% scenes
for s = 1:length(scenes)
scene = scenes{s};
sceneName = strrep(scene,'_','-');
modelPath = sprintf('%s/%s/%s-mesh.ply',resDir,subName,subName);
%%
if 0
%% read models
model = read_model(modelPath);
figure(101); clf; trimesh(model.tri,model.vtx(:,1),model.vtx(:,2),model.vtx(:,3)); axis equal; title('submitted');
end
%% project to images
stats.conf = zeros(labelCount-1,labelCount-1,length(evalCams));
stats.acc = zeros(length(evalCams),length(evalFrames));
for c = 1:length(evalCams)
%% select camera
idCam = evalCams(c);
acc = zeros(length(evalFrames),1);
conf = zeros(labelCount-1,labelCount-1,length(evalFrames));
vid = VideoWriter(sprintf('%s/%s/%s/sem-vid-cam%d.avi',resDir,subName,scene,idCam));
vid.FrameRate = 3;
vid.Quality = 100;
open(vid);
for i = 1:length(evalFrames)
%% select frame
idFrame = evalFrames(i);
basePath = sprintf('%s/%s/%s_%d/%s_%d_f%05d',dataPath,scene,camType,idCam,camType,idCam,idFrame);
txtPath = [basePath '_cam.txt'];
clsPath = sprintf('%s/%s/%s/%s_%d/%s_%d_f%05d_undist.png',resDir,subName,scene,camType,idCam,camType,idCam,idFrame);
if exist(txtPath,'file')
% %% read from colmap txt
% txtCam = load(txtPath);
% cam.f = txtCam(1:2); % fx fy
% cam.c = txtCam(3:4); % cx cy
% cam.q = txtCam(5:8); % qw qx qy qz
% cam.t = txtCam(9:11); % tx ty tz
% cam.resolution = [752, 480];
%
% cam.R = quat2rotm(cam.q);
% cam.K = eye(3);
% cam.K(1,1) = cam.f(1);
% cam.K(2,2) = cam.f(2);
% cam.K(1,3) = cam.c(1);
% cam.K(2,3) = cam.c(2);
% fprintf('Cam pose loaded from %s\n',txtPath);
%
% %% mesh projection
% camParsMesh{1} = struct('TcV', cam.t, ...
% 'RcM', cam.R, ...
% 'fcV', [cam.K(1,1); cam.K(2,2)], ...
% 'ccV', [cam.K(1,3); cam.K(2,3)], ...
% 'imSizeV', [cam.resolution(2); cam.resolution(1)]);
% projZrange = [1e-3; 2000];
% %% gt depths
% [projDmap, cx] = RenderDepthMesh(gtmodel.tri, gtmodel.ptXh(:,1:3), camParsMesh{1}, ...
% [cam.resolution(2); cam.resolution(1)], projZrange, 1, 0);
if exist(clsPath,'file')
%% load anot from file
imgAnot = imread(clsPath);
imgAnot = flip(imgAnot,2);
else
projColor = RenderColorMesh(model.tri, model.ptXh(:,1:3), single(model.vtxColor)/255, ...
camParsMesh{1}, [cam.resolution(2); cam.resolution(1)], projZrange, 1);
%% tranform color to labels
projColor = uint8(projColor);
imgAnot = uint8(rgbmapind(projColor,uint8(def.labelColors*255))) - 1;
end
imgAnot(imgAnot(:)==0) = 9;
else
stats.acc(c,i) = nan;
continue;
end
%% load gt anot
gtAnot = imread([basePath '_gtr.png']);
gtAnot(gtAnot(:)>10) = 9;
gtAnotTest = gtAnot;
gtValid = true(size(gtAnot));
% gtValid = (projDmap<1);
% gtValid = imdilate(gtValid,strel('diamond',3));
gtValid(gtAnot(:)==0) = 0;
gtAnotTest(gtValid==0) = 0;
gtDiff = double(gtAnotTest~=imgAnot);
gtDiff(gtAnotTest==0) = -1;
%% stats
stats.acc(c,i) = sum(gtDiff(:)==0) / sum(gtDiff(:)>=0);
conf(:,:,i) = confMatrix(gtAnotTest(gtValid(:)),imgAnot(gtValid(:)),labelCount-1);
%figure(103);
%confMatrixShow(conf(:,:,i), def.labelNames(2:end), {'FontSize',12}, 2, 1 ); colormap hot; ylabel('GT');
%% plot
figure(1);
subplot(2,2,1); imshow(imgAnot,def.labelColors); axis image; title(sprintf('cam %d frame %d: submitted labels',idCam,idFrame));
subplot(2,2,2); % imagesc(projDmap); axis image; title('gt depths');
imagesc(max(conf(:))-conf(:,:,i)); axis image; colormap hot; ylabel('GT'); title('confusion matrix');
subplot(2,2,3); imshow(gtAnotTest,def.labelColors); axis image; title('gt labels (masked)');
subplot(2,2,4); imshow(gtDiff+2,[0 0 0; 0.5 0.5 0.5; 1 0 0]); axis image; title(sprintf('error mask (accuracy = %.03f)',stats.acc(c,i))); %colormap jet;
drawnow;
writeVideo(vid,getframe(gcf));
end
close(vid);
%% camera totals
camconf = sum(conf,3);
stats.conf(:,:,c) = sum(camconf,3);
stats.cacc(c) = sum(diag(camconf))/sum(camconf(:));
mstats.acc(s,c) = stats.cacc(c);
end
%% scene totals
resfn = sprintf('%s/%s/%s-sem',resDir,subName,scene);
stats.tconf = sum(stats.conf,3);
stats.tacc = sum(diag(stats.tconf))/sum(stats.tconf(:));
disp(stats.tacc);
save([resfn '-stats.mat'],'-struct','stats');
%
figure(105);
plot(stats.acc','*'); ylabel 'accuracy'; grid on; xlabel('frame');
title(sprintf('%s[%s]: total pixelwise accuracy = %.3f',subName,sceneName,stats.tacc));
print([resfn '-acc.pdf'],'-dpdf','-bestfit');
%
figure(106);
confMatrixShow(stats.tconf, def.labelNames(2:end), {'FontSize',12}, 2, 1 );
colormap hot; ylabel('GT');
title(sprintf('%s[%s]: total pixelwise accuracy = %.3f',subName,sceneName,stats.tacc));
print([resfn '-conf.pdf'],'-dpdf','-bestfit');
%% main stats
mstats.conf(:,:,s) = stats.tconf;
mstats.cacc(c) = stats.tacc;
end
%% submission totals
resfn = sprintf('%s/%s/%s-sem-all',resDir,subName,resName);
mstats.name = subTitle;
mstats.tconf = sum(mstats.conf,3);
mstats.tacc = sum(diag(mstats.conf))/sum(mstats.conf(:));
disp(mstats.tacc);
save([resfn '-stats.mat'],'-struct','stats');
%
figure(105);
plot(mstats.acc','*'); ylabel 'accuracy'; grid on; xlabel('camera');
title(sprintf('%s[%s]: total pixelwise accuracy = %.3f',subTitle,resName,mstats.tacc));
print([resfn '-acc.pdf'],'-dpdf','-bestfit');
%
figure(106);
confMatrixShow(mstats.tconf, def.labelNames(2:end), {'FontSize',12}, 2, 1 );
colormap hot; ylabel('GT');
title(sprintf('%s[%s]: total pixelwise accuracy = %.3f',subTitle,resName,mstats.tacc));
print([resfn '-conf.pdf'],'-dpdf','-bestfit');
end
#!/bin/bash
CC="cloudcompare.CloudCompare"
GT="gt/model_10mm_$2.ply"
echo "*** Computing accuracy ***"
$CC -silent -auto_save OFF -C_EXPORT_FMT PLY \
-o $1 \
-crop -2:-2:-1:14:14:7 \
-SS SPATIAL 0.01 \
-o $GT \
-REMOVE_ALL_SFS \
-c2c_dist \
-pop_clouds \
-SF_COLOR_SCALE cmap.xml \
-SF_CONVERT_TO_RGB false \
-SS SPATIAL 0.01 \
-save_clouds file "${1%.*}-acc.ply"
echo "*** Computing completeness ***"
$CC -silent -auto_save OFF -C_EXPORT_FMT PLY \
-o $GT \
-o $1 \
-crop -2:-2:-1:14:14:7 \
-REMOVE_ALL_SFS \
-c2c_dist \
-pop_clouds \
-SF_COLOR_SCALE cmap.xml \
-SF_CONVERT_TO_RGB false \
-save_clouds file "${1%.*}-comp.ply"
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018/training/model_10mm_0001.ply
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018/training/model_10mm_0128.ply
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018/training/model_10mm_0160.ply
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018/training/model_10mm_0224.ply
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018eval/gt/model_10mm_0288.ply
\ No newline at end of file
/home/radim/Documents/proj/TrimBot/Challenge2018eval/gt/model_10mm_real.ply
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
function R = quat2rotm(quat)
% x y z w
quat = quat/norm(quat);
w = quat(1);
x = quat(2);
y = quat(3);
z = quat(4);
R = [1 - 2*y*y - 2*z*z, 2*x*y - 2*z*w, 2*x*z + 2*y*w;
2*x*y + 2*z*w, 1 - 2*x*x - 2*z*z, 2*y*z - 2*x*w;
2*x*z - 2*y*w, 2*y*z + 2*x*w, 1 - 2*x*x - 2*y*y];
\ No newline at end of file
function def = read_labels(dir)
%READLABELS Read semantic shape labels and colors into handles struct
% from yaml files in dir
% labelNames
% labelIDs
% labelColors
%% read labels
def.labels = YAML.read(fullfile(dir, 'labels.yaml'));
def.colors = YAML.read(fullfile(dir, 'colors.yaml'));
groupNames = fieldnames(def.labels.Semantic);
id = 1;
for g = 1:length(groupNames)
gn = groupNames{g};
gel = def.labels.Semantic.(gn);
if isstruct(gel)
groupLabels = fieldnames(gel);
for gi = 1:length(groupLabels)
gl = groupLabels{gi};
def.labelNames{id} = [gn '-' gl];
def.labelIDs(id) = def.labels.Semantic.(gn).(gl);
def.labelColors(id,1:3) = def.colors.Semantic.(gn).(gl);
id = id + 1;
end
else
def.labelNames{id} = gn;
def.labelIDs(id) = def.labels.Semantic.(gn);
def.labelColors(id,1:3) = def.colors.Semantic.(gn);
id = id + 1;
end
end
%% mapping
if isfield(def.labels,'Mapping')
def.mapping = zeros(255,1);
groupNames = fieldnames(def.labels.Mapping);
for g = 1:length(groupNames)
gn = groupNames{g};
gel = def.labels.Mapping.(gn);
if isstruct(gel)
groupLabels = fieldnames(gel);
for gi = 1:length(groupLabels)
gl = groupLabels{gi};
lmap = def.labels.Mapping.(gn).(gl);
def.mapping(lmap(1)) = lmap(2);
end
end
end
end
\ No newline at end of file
function model = read_model(modelPath)
%% read mesh model
fprintf('Reading model %s ... ',modelPath); tic;
[tri, vtx, model.data, model.name] = ply_read(modelPath,'tri');
model.vtx = vtx';
model.tri = tri';
if isfield(model.data.vertex,'red')
model.vtxColor = [model.data.vertex.red model.data.vertex.green model.data.vertex.blue];
else
model.vtxColor = [];
end
model.vtxLabels = []; %model.data.vertex.alpha;
model.triColor = [];
%% color point cloud matrix
%model.ptXcol = [model.vtx model.vtxColor];
model.ptXh = [model.vtx, ones(size(model.vtx,1),1)];
%model.ptBox = [min(model.vtx,[],1); max(model.vtx,[],1)];
toc;
\ No newline at end of file
%% function [FaceColorT, FaceVisibleM] = BatchFaceColorGrad(FM, VM, CamParamSA, ...
% ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV)
%
% This function will use OpenGL to render a set of triangular faces (defined by FM and VM arrays)
%
function [FaceColorT, FaceVisibleM] = BatchFaceColorGrad(FM, VM, CamParamSA, ...
ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV)
MexGlutInit;
%% Check the input data type
imageNum = length(TestImageA);
for ii = 1 : imageNum
if (~isa(TestImageA{ii}, 'uint8'))
error('Offscreen:BatchFaceColorGrad:Wrong Input Type', 'Input image should be uint8');
end
if (~isa(ImageMaskA{ii}, 'uint8'))
error('Offscreen:BatchFaceColorGrad:Wrong Input Type', 'Mask image should be uint8');
end
end
if (~isa(FM, 'double'))
FM = double(FM);
end
[FaceColorT, FaceVisibleM] = BatchFaceColorGradImpl(FM, VM, CamParamSA, ...
ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV);
%% Wish 1: Decompose matlab camera parameter to different vectors
function [ViewPointV, LookAtV, UpV, fcV, ccV] = CamS2Vector1(CamParamS)
InvRcM = inv(CamParamS.RcM);
%% compute the viewpoint
ViewPointV = InvRcM * (-CamParamS.TcV);
%% the lookat vector is the camera's z axis
LookAtV = InvRcM * [0; 0; 1];
%% the Up vector is the camera's -y axis
UpV = InvRcM * [0; -1; 0];
fcV = CamParamS.fcV;
ccV = CamParamS.ccV;
%%
% function CamParamS = CamSInterp1(CamParam1S, CamParam2S, lamda)
% return a camera paramter that is an interpolated version of the two
% CamParamS = lamda * CamParam1S + (1-lamda) * CamParam2S