Commit 6cd0851c authored by Radim Tylecek's avatar Radim Tylecek

evaluation added

parent 669e2bd0
*.ply filter=lfs diff=lfs merge=lfs -text
*.avi filter=lfs diff=lfs merge=lfs -text
classdef YAML
%YAML Serialize a matlab variable to yaml format
%
% [ X ] = YAML.load( S )
% [ S ] = YAML.dump( X )
%
% [ X ] = YAML.read( filepath )
% YAML.write( filepath, X )
%
% YAML.LOAD takes YAML string S and returns matlab variable X.
% YAML.DUMP takes matlab variable X and converts to YAML string S.
% YAML.READ and YAML.WRITE are convenient methods to load and dump
% YAML format directly from a file.
%
% Examples:
% To serialize matlab object
%
% >> X = struct('matrix', rand(3,4), 'char', 'hello');
% >> S = YAML.dump(X);
% >> disp(S);
% matrix:
% - [0.9571669482429456, 0.14188633862721534]
% - [0.4853756487228412, 0.421761282626275]
% - [0.8002804688888001, 0.9157355251890671]
% char: hello
%
% To decode yaml string
%
% >> X = YAML.load(S);
% >> disp(X)
% matrix: [3x2 double]
% char: 'hello'
%
% See also: xmlread xmlwrite
properties (Constant)
JARFILE = YAML.jarfile
end
methods (Static)
function [ S ] = jarfile()
%JARFILE path to the SnakeYAML jar file
S = fileparts(mfilename('fullpath'));
S = [S filesep 'java' filesep 'snakeyaml-1.9.jar'];
end
function [ X ] = load( S )
%LOAD load matlab object from yaml string
javaaddpath(YAML.JARFILE);
% Load yaml into java obj
yaml = org.yaml.snakeyaml.Yaml;
java_obj = yaml.load(S);
% Convert to matlab object
X = YAML.load_data(java_obj);
end
function [ S ] = dump( X )
%DUMP serialize matlab object into yaml string
javaaddpath(YAML.JARFILE);
% Convert matlab obj to java obj
yaml = org.yaml.snakeyaml.Yaml();
java_obj = YAML.dump_data(X);
% Dump into yaml string
S = char(yaml.dump(java_obj));
end
function [ X ] = read( filepath )
%READ read and decode yaml data from file
fid = fopen(filepath,'r');
S = fscanf(fid,'%c',inf);
fclose(fid);
X = YAML.load( S );
end
function [] = write( filepath, X )
%WRITE serialize and write yaml data to file
S = YAML.dump( X );
fid = fopen(filepath,'w');
fprintf(fid,'%s',S);
fclose(fid);
end
end
methods(Static, Access=private)
function result = load_data( r )
%LOAD_DATA recursively convert java objects
if isa(r, 'char')
result = char(r);
elseif isa(r, 'double')
result = double(r);
elseif isa(r, 'java.util.Date')
result = DateTime(r);
elseif isa(r, 'java.util.List')
result = cell(r.size(),1);
itr = r.iterator();
i = 1;
while itr.hasNext()
result{i} = YAML.load_data(itr.next());
i = i + 1;
end
result = YAML.merge_cell(result);
elseif isa(r, 'java.util.Map')
result = struct;
itr = r.keySet().iterator();
while itr.hasNext()
key = itr.next();
result.(char(key)) = YAML.load_data(...
r.get(java.lang.String(key)));
end
else
error('YAML:load_data:typeError',...
['Unknown data type: ' class(r)]);
end
end
function result = merge_cell( r )
%MERGE_CELL convert cell array to native matrix
% Check eligibility
merge = false;
if all(cellfun(@isnumeric,r))
merge = true;
elseif all(cellfun(@isstruct,r))
f = cellfun(@fieldnames,r,'UniformOutput',false);
if isempty(f) || (numel(unique(cellfun(@numel,f)))==1 && ...
all(cellfun(@(x) all(strcmp(f{1},x)),f)))
merge = true;
end
end
% Merge if scalar or row vector
result = r;
if merge
if all(cellfun(@isscalar,r))
result = [r{:}];
elseif all(cellfun(@isrow,r)) &&...
length(unique(cellfun(@length,r)))==1
result = cat(1,r{:});
end
end
end
function result = dump_data( r )
%DUMP_DATA convert
if ischar(r)
result = java.lang.String(r);
elseif ~isscalar(r)
result = java.util.ArrayList();
if size(r,1)==1
for i = 1:numel(r)
if iscell(r)
result.add(YAML.dump_data(r{i}));
else
result.add(YAML.dump_data(r(i)));
end
end
else
for i = 1:size(r,1)
result.add(YAML.dump_data(r(i,:)));
end
end
elseif isnumeric(r)
result = java.lang.Double(r);
elseif isstruct(r)
result = java.util.LinkedHashMap();
keys = fields(r);
for i = 1:length(keys)
result.put(keys{i},YAML.dump_data(r.(keys{i})));
end
elseif iscell(r)
result = java.util.ArrayList();
result.add(YAML.dump_data(r{1}));
elseif isa(r,'DateTime')
result = java.util.Date(datestr(r));
else
error('YAML:load_data:typeError',...
['Unsupported data type: ' class(r)]);
end
end
end
end
function stats = eval_recon(subName, accRatio, compThreshold, histBins, maxRange)
%% EVAL_RECON - evaluate accuracy and completeness of a model
% acc is distance d (in m) such that [accRatio]% of the reconstruction is within d of the ground truth mesh
% comp is the percent of points on the GTM that are within [compThreshold] mm of the reconstruction
accPath = sprintf('submissions/%s/%s-acc.ply',subName,subName); % distances to GT
compPath = sprintf('submissions/%s/%s-comp.ply',subName,subName); % distances from GT
stats.accRatio = accRatio; %0.9;
stats.compThreshold = compThreshold; %0.1;
% histBins = 10;
%% read distances from CloudCompare pcls
compPcl = ply_read(compPath);
compDist = abs(compPcl.vertex.('scalar_C2M'));
accPcl = ply_read(accPath);
accDist = abs(accPcl.vertex.('scalar_C2C'));
%% compute stats
stats.acc = quantile(accDist,accRatio);
stats.comp = mean(compDist<compThreshold);
stats.histBins = linspace(maxRange/histBins/2, maxRange-maxRange/histBins/2, histBins);
stats.histBins = maxRange*logspace(-2,0,histBins);
histComp = hist(compDist,stats.histBins);
stats.histComp = histComp/sum(histComp);
histAcc = hist(accDist,stats.histBins);
stats.histAcc = histAcc/sum(histAcc);
%% plot
figure('Name',subName);
subplot(2,2,1);
bar(stats.histBins,cumsum(stats.histComp)); title(sprintf('completeness: %.1f %%',stats.comp*100)); xlabel('m (GT to X)'); ylim([0 1]);
hold on; plot(compThreshold*[1 1],[0 1],'r-');
subplot(2,2,2);
bar(stats.histBins,cumsum(stats.histAcc)); title(sprintf('accuracy: %.3f m',stats.acc)); xlabel('m (X to GT)'); ylim([0 1]);
hold on; plot([0 maxRange],accRatio*[1 1],'r-');
subplot(2,2,3); imshow(imread(sprintf('submissions/%s/%s-comp.png',subName,subName)));
subplot(2,2,4); imshow(imread(sprintf('submissions/%s/%s-acc.png',subName,subName)));
print(sprintf('submissions/%s-recon.pdf',subName),'-dpdf','-bestfit');
%% evaluate semantics
addpath('render','toolbox');
subName = 'jm';
modelPath = sprintf('submissions/%s/%s-mesh.ply',subName,subName); % distances to GT
dataPath = '../testing/test_around_garden';
gtPath = 'gt/mesh-crop.ply';
evalFrames = 140:10:1480;
evalCams = [0 2];
%% read label def
def = read_labels('../calibration');
labelCount = length(def.labelNames);
if 1
%% read models
model = read_model(modelPath);
figure(101); clf; trimesh(model.tri,model.vtx(:,1),model.vtx(:,2),model.vtx(:,3)); axis equal; title('submitted');
end
gtmodel = read_model(gtPath);
figure(102); clf; trimesh(gtmodel.tri,gtmodel.vtx(:,1),gtmodel.vtx(:,2),gtmodel.vtx(:,3)); axis equal; title('gt');
%% project to images
stats.conf = zeros(labelCount-1,labelCount-1,length(evalCams));
stats.acc = zeros(length(evalCams),length(evalFrames));
for c = 1:length(evalCams)
%% select camera
idCam = evalCams(c);
acc = zeros(length(evalFrames),1);
conf = zeros(labelCount-1,labelCount-1,length(evalFrames));
vid = VideoWriter(sprintf('submissions/%s-sem-vid-cam%d.avi',subName,idCam));
vid.FrameRate = 3;
vid.Quality = 100;
open(vid);
for i = 1:length(evalFrames)
%% select frame
idFrame = evalFrames(i);
basePath = sprintf('%s/uvc_camera_cam_%d/uvc_camera_cam_%d_f%05d',dataPath,idCam,idCam,idFrame);
txtPath = [basePath '_cam.txt'];
clsPath = sprintf('submissions/%s/uvc_camera_cam_%d/uvc_camera_cam_%d_f%05d_undist.png',subName,idCam,idCam,idFrame);
if exist(txtPath,'file')
%% read from colmap txt
txtCam = load(txtPath);
cam.f = txtCam(1:2); % fx fy
cam.c = txtCam(3:4); % cx cy
cam.q = txtCam(5:8); % qw qx qy qz
cam.t = txtCam(9:11); % tx ty tz
cam.resolution = [752, 480];
cam.R = quat2rotm(cam.q);
cam.K = eye(3);
cam.K(1,1) = cam.f(1);
cam.K(2,2) = cam.f(2);
cam.K(1,3) = cam.c(1);
cam.K(2,3) = cam.c(2);
fprintf('Cam pose loaded from %s\n',txtPath);
%% mesh projection
camParsMesh{1} = struct('TcV', cam.t, ...
'RcM', cam.R, ...
'fcV', [cam.K(1,1); cam.K(2,2)], ...
'ccV', [cam.K(1,3); cam.K(2,3)], ...
'imSizeV', [cam.resolution(2); cam.resolution(1)]);
projZrange = [1e-3; 2000];
%% gt depths
[projDmap, cx] = RenderDepthMesh(gtmodel.tri, gtmodel.ptXh(:,1:3), camParsMesh{1}, ...
[cam.resolution(2); cam.resolution(1)], projZrange, 1, 0);
if exist(clsPath,'file')
%% load anot from file
imgAnot = imread(clsPath);
else
projColor = RenderColorMesh(model.tri, model.ptXh(:,1:3), single(model.vtxColor)/255, ...
camParsMesh{1}, [cam.resolution(2); cam.resolution(1)], projZrange, 1);
%% tranform color to labels
projColor = uint8(projColor);
imgAnot = uint8(rgbmapind(projColor,uint8(def.labelColors*255))) - 1;
end
imgAnot(imgAnot(:)==0) = 9;
else
stats.acc(c,i) = nan;
continue;
end
%% load gt anot
gtAnot = imread([basePath '_gtr.png']);
gtAnot(gtAnot(:)>10) = 9;
gtAnotTest = gtAnot;
gtValid = (projDmap<1);
gtValid = imdilate(gtValid,strel('diamond',3));
gtValid(gtAnot(:)==0) = 0;
gtAnotTest(gtValid==0) = 0;
gtDiff = double(gtAnotTest~=imgAnot);
gtDiff(gtAnotTest==0) = -1;
%% stats
stats.acc(c,i) = sum(gtDiff(:)==0) / sum(gtDiff(:)>=0);
conf(:,:,i) = confMatrix(gtAnotTest(gtValid(:)),imgAnot(gtValid(:)),labelCount-1);
%figure(103);
%confMatrixShow(conf(:,:,i), def.labelNames(2:end), {'FontSize',12}, 2, 1 ); colormap hot; ylabel('GT');
% plot
figure(1);
subplot(2,2,1); imshow(imgAnot,def.labelColors); axis image; title(sprintf('cam %d frame %d: submitted labels',idCam,idFrame));
subplot(2,2,2); imagesc(projDmap); axis image; title('gt depths');
subplot(2,2,3); imshow(gtAnotTest,def.labelColors); axis image; title('gt labels (masked)');
subplot(2,2,4); imshow(gtDiff+2,[0 0 0; 0.5 0.5 0.5; 1 0 0]); axis image; title(sprintf('error mask (accuracy = %.03f)',stats.acc(c,i))); colormap jet;
drawnow;
writeVideo(vid,getframe(gcf));
end
close(vid);
%% camera totals
camconf = sum(conf,3);
stats.conf(:,:,c) = sum(camconf,3);
stats.cacc(c) = sum(diag(camconf))/sum(camconf(:));
end
%% overall totals
stats.tconf = sum(stats.conf,3);
stats.tacc = sum(diag(stats.tconf))/sum(stats.tconf(:));
disp(stats.tacc);
save(sprintf('submissions/%s/stats.mat',subName),'-struct','stats');
%
figure(105);
plot(stats.acc','*'); ylabel 'accuracy'; grid on;
title(sprintf('%s: total pixelwise accuracy = %.3f',subName,stats.tacc));
print(sprintf('submissions/%s-acc.pdf',subName),'-dpdf','-bestfit');
%
figure(106);
confMatrixShow(stats.tconf, def.labelNames(2:end), {'FontSize',12}, 2, 1 );
colormap hot; ylabel('GT');
title(sprintf('%s: total pixelwise accuracy = %.3f',subName,stats.tacc));
print(sprintf('submissions/%s-confmat.pdf',subName),'-dpdf','-bestfit');
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
function R = quat2rotm(quat)
% x y z w
quat = quat/norm(quat);
w = quat(1);
x = quat(2);
y = quat(3);
z = quat(4);
R = [1 - 2*y*y - 2*z*z, 2*x*y - 2*z*w, 2*x*z + 2*y*w;
2*x*y + 2*z*w, 1 - 2*x*x - 2*z*z, 2*y*z - 2*x*w;
2*x*z - 2*y*w, 2*y*z + 2*x*w, 1 - 2*x*x - 2*y*y];
\ No newline at end of file
function def = read_labels(dir)
%READLABELS Read semantic shape labels and colors into handles struct
% from yaml files in dir
% labelNames
% labelIDs
% labelColors
%% read labels
def.labels = YAML.read(fullfile(dir, 'labels.yaml'));
def.colors = YAML.read(fullfile(dir, 'colors.yaml'));
groupNames = fieldnames(def.labels.Semantic);
id = 1;
for g = 1:length(groupNames)
gn = groupNames{g};
gel = def.labels.Semantic.(gn);
if isstruct(gel)
groupLabels = fieldnames(gel);
for gi = 1:length(groupLabels)
gl = groupLabels{gi};
def.labelNames{id} = [gn '-' gl];
def.labelIDs(id) = def.labels.Semantic.(gn).(gl);
def.labelColors(id,1:3) = def.colors.Semantic.(gn).(gl);
id = id + 1;
end
else
def.labelNames{id} = gn;
def.labelIDs(id) = def.labels.Semantic.(gn);
def.labelColors(id,1:3) = def.colors.Semantic.(gn);
id = id + 1;
end
end
%% mapping
if isfield(def.labels,'Mapping')
def.mapping = zeros(255,1);
groupNames = fieldnames(def.labels.Mapping);
for g = 1:length(groupNames)
gn = groupNames{g};
gel = def.labels.Mapping.(gn);
if isstruct(gel)
groupLabels = fieldnames(gel);
for gi = 1:length(groupLabels)
gl = groupLabels{gi};
lmap = def.labels.Mapping.(gn).(gl);
def.mapping(lmap(1)) = lmap(2);
end
end
end
end
\ No newline at end of file
function model = read_model(modelPath)
%% read mesh model
fprintf('Reading model %s ... ',modelPath); tic;
[tri, vtx, model.data, model.name] = ply_read(modelPath,'tri');
model.vtx = vtx';
model.tri = tri';
if isfield(model.data.vertex,'red')
model.vtxColor = [model.data.vertex.red model.data.vertex.green model.data.vertex.blue];
else
model.vtxColor = [];
end
model.vtxLabels = []; %model.data.vertex.alpha;
model.triColor = [];
%% color point cloud matrix
%model.ptXcol = [model.vtx model.vtxColor];
model.ptXh = [model.vtx, ones(size(model.vtx,1),1)];
%model.ptBox = [min(model.vtx,[],1); max(model.vtx,[],1)];
toc;
\ No newline at end of file
%% function [FaceColorT, FaceVisibleM] = BatchFaceColorGrad(FM, VM, CamParamSA, ...
% ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV)
%
% This function will use OpenGL to render a set of triangular faces (defined by FM and VM arrays)
%
function [FaceColorT, FaceVisibleM] = BatchFaceColorGrad(FM, VM, CamParamSA, ...
ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV)
MexGlutInit;
%% Check the input data type
imageNum = length(TestImageA);
for ii = 1 : imageNum
if (~isa(TestImageA{ii}, 'uint8'))
error('Offscreen:BatchFaceColorGrad:Wrong Input Type', 'Input image should be uint8');
end
if (~isa(ImageMaskA{ii}, 'uint8'))
error('Offscreen:BatchFaceColorGrad:Wrong Input Type', 'Mask image should be uint8');
end
end
if (~isa(FM, 'double'))
FM = double(FM);
end
[FaceColorT, FaceVisibleM] = BatchFaceColorGradImpl(FM, VM, CamParamSA, ...
ScreenSizeV, zoomFactor, TestImageA, ImageMaskA, channelModFactor, zNearFarV);
%% Wish 1: Decompose matlab camera parameter to different vectors
function [ViewPointV, LookAtV, UpV, fcV, ccV] = CamS2Vector1(CamParamS)
InvRcM = inv(CamParamS.RcM);
%% compute the viewpoint
ViewPointV = InvRcM * (-CamParamS.TcV);
%% the lookat vector is the camera's z axis
LookAtV = InvRcM * [0; 0; 1];
%% the Up vector is the camera's -y axis
UpV = InvRcM * [0; -1; 0];
fcV = CamParamS.fcV;
ccV = CamParamS.ccV;
%%
% function CamParamS = CamSInterp1(CamParam1S, CamParam2S, lamda)
% return a camera paramter that is an interpolated version of the two
% CamParamS = lamda * CamParam1S + (1-lamda) * CamParam2S
function CamParamS = CamSInterp1(CamParam1S, CamParam2S, lamda)
[ViewPoint1V, LookAt1V, Up1V, fc1V, cc1V] = CamS2Vector1(CamParam1S