@conference {Kluger2020,
	title = {CONSAC: Robust Multi-Model Fitting by Conditional Sample Consensus},
	booktitle = {CVPR 2020},
	year = {2020},
	month = {01/2020},
	abstract = {We present a robust estimator for fitting multiple parametric models of the same form to noisy measurements. Applications include finding multiple vanishing points in man-made scenes, fitting planes to architectural imagery, or estimating multiple rigid motions within the same sequence. In contrast to previous works, which resorted to hand-crafted search strategies for multiple model detection, we learn the search strategy from data. A neural network conditioned on previously detected models guides a RANSAC estimator to different subsets of all measurements, thereby finding model instances one after another. We train our method supervised as well as self-supervised. For supervised training of the search strategy, we contribute a new dataset for vanishing point estimation. Leveraging this dataset, the proposed algorithm is superior with respect to other robust estimators as well as to designated vanishing point estimation algorithms. For self-supervised learning of the search, we evaluate the proposed algorithm on multi-homography estimation and demonstrate an accuracy that is superior to state-of-the-art methods.},
	url = {http://arxiv.org/abs/2001.02643},
	author = {Kluger, Florian and Brachmann, Eric and Ackermann, Hanno and Carsten Rother and Yang, Michael Ying and Rosenhahn, Bodo}
}
@conference {Bhowmik2019,
	title = {Reinforced Feature Points: Optimizing Feature Detection and Description for a High-Level Task},
	booktitle = {CVPR 2020 (oral)},
	year = {2020},
	abstract = {We address a core problem of computer vision: Detection and description of 2D feature points for image matching. For a long time, hand-crafted designs, like the seminal SIFT algorithm, were unsurpassed in accuracy and efficiency. Recently, learned feature detectors emerged that implement detection and description using neural networks. Training these networks usually resorts to optimizing low-level matching scores, often pre-defining sets of image patches which should or should not match, or which should or should not contain key points. Unfortunately, increased accuracy for these low-level matching scores does not necessarily translate to better performance in high-level vision tasks. We propose a new training methodology which embeds the feature detector in a complete vision pipeline, and where the learnable parameters are trained in an end-to-end fashion. We overcome the discrete nature of key point selection and descriptor matching using principles from reinforcement learning. As an example, we address the task of relative pose estimation between a pair of images. We demonstrate that the accuracy of a state-of-the-art learning-based feature detector can be increased when trained for the task it is supposed to solve at test time. Our training methodology poses little restrictions on the task to learn, and works for any architecture which predicts key point heat maps, and descriptors for key point locations.},
	url = {http://arxiv.org/abs/1912.00623},
	author = {Bhowmik, Aritra and Gumhold, Stefan and Rother, Carsten and Brachmann, Eric}
}
@conference {Brachmann2019,
	title = {Expert sample consensus applied to camera re-localization},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2019-Octob},
	year = {2019},
	month = {aug},
	pages = {7524{\textendash}7533},
	abstract = {Fitting model parameters to a set of noisy data points is a common problem in computer vision. In this work, we fit the 6D camera pose to a set of noisy correspondences between the 2D input image and a known 3D environment. We estimate these correspondences from the image using a neural network. Since the correspondences often contain outliers, we utilize a robust estimator such as Random Sample Consensus (RANSAC) or Differentiable RANSAC (DSAC) to fit the pose parameters. When the problem domain, e.g. the space of all 2D-3D correspondences, is large or ambiguous, a single network does not cover the domain well. Mixture of Experts (MoE) is a popular strategy to divide a problem domain among an ensemble of specialized networks, so called experts, where a gating network decides which expert is responsible for a given input. In this work, we introduce Expert Sample Consensus (ESAC), which integrates DSAC in a MoE. Our main technical contribution is an efficient method to train ESAC jointly and end-to-end. We demonstrate experimentally that ESAC handles two real-world problems better than competing methods, i.e. scalability and ambiguity. We apply ESAC to fitting simple geometric models to synthetic images, and to camera re-localization for difficult, real datasets.},
	isbn = {9781728148038},
	issn = {15505499},
	doi = {10.1109/ICCV.2019.00762},
	url = {http://arxiv.org/abs/1908.02484},
	author = {Brachmann, Eric and Carsten Rother}
}
@conference {HosseiniJafari2019,
	title = {iPose: Instance-Aware 6D Pose Estimation of Partly Occluded Objects},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11363 LNCS},
	year = {2019},
	pages = {477{\textendash}492},
	abstract = {We address the task of 6D pose estimation of known rigid objects from single input images in scenarios where the objects are partly occluded. Recent RGB-D-based methods are robust to moderate degrees of occlusion. For RGB inputs, no previous method works well for partly occluded objects. Our main contribution is to present the first deep learning-based system that estimates accurate poses for partly occluded objects from RGB-D and RGB input. We achieve this with a new instance-aware pipeline that decomposes 6D object pose estimation into a sequence of simpler steps, where each step removes specific aspects of the problem. The first step localizes all known objects in the image using an instance segmentation network, and hence eliminates surrounding clutter and occluders. The second step densely maps pixels to 3D object surface positions, so called object coordinates, using an encoder-decoder network, and hence eliminates object appearance. The third, and final, step predicts the 6D pose using geometric optimization. We demonstrate that we significantly outperform the state-of-the-art for pose estimation of partly occluded objects for both RGB and RGB-D input.},
	isbn = {9783030208929},
	issn = {16113349},
	doi = {10.1007/978-3-030-20893-6_30},
	author = {Omid Hosseini Jafari and Mustikovela, Siva Karthik and Pertsch, Karl and Brachmann, Eric and Carsten Rother}
}
@conference {Brachmann2019a,
	title = {Neural-guided RANSAC: Learning where to sample model hypotheses},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2019-Octob},
	year = {2019},
	month = {may},
	pages = {4321{\textendash}4330},
	abstract = {We present Neural-Guided RANSAC (NG-RANSAC), an extension to the classic RANSAC algorithm from robust optimization. NG-RANSAC uses prior information to improve model hypothesis search, increasing the chance of finding outlier-free minimal sets. Previous works use heuristic side-information like hand-crafted descriptor distance to guide hypothesis search. In contrast, we learn hypothesis search in a principled fashion that lets us optimize an arbitrary task loss during training, leading to large improvements on classic computer vision tasks. We present two further extensions to NG-RANSAC. Firstly, using the inlier count itself as training signal allows us to train neural guidance in a self-supervised fashion. Secondly, we combine neural guidance with differentiable RANSAC to build neural networks which focus on certain parts of the input data and make the output predictions as good as possible. We evaluate NG-RANSAC on a wide array of computer vision tasks, namely estimation of epipolar geometry, horizon line estimation and camera re-localization. We achieve superior or competitive results compared to state-of-the-art robust estimators, including very recent, learned ones.},
	isbn = {9781728148038},
	issn = {15505499},
	doi = {10.1109/ICCV.2019.00442},
	url = {http://arxiv.org/abs/1905.04132},
	author = {Brachmann, Eric and Carsten Rother}
}
@article {Bhowmik2019,
	title = {Reinforced Feature Points: Optimizing Feature Detection and Description for a High-Level Task},
	year = {2019},
	month = {dec},
	abstract = {We address a core problem of computer vision: Detection and description of 2D feature points for image matching. For a long time, hand-crafted designs, like the seminal SIFT algorithm, were unsurpassed in accuracy and efficiency. Recently, learned feature detectors emerged that implement detection and description using neural networks. Training these networks usually resorts to optimizing low-level matching scores, often pre-defining sets of image patches which should or should not match, or which should or should not contain key points. Unfortunately, increased accuracy for these low-level matching scores does not necessarily translate to better performance in high-level vision tasks. We propose a new training methodology which embeds the feature detector in a complete vision pipeline, and where the learnable parameters are trained in an end-to-end fashion. We overcome the discrete nature of key point selection and descriptor matching using principles from reinforcement learning. As an example, we address the task of relative pose estimation between a pair of images. We demonstrate that the accuracy of a state-of-the-art learning-based feature detector can be increased when trained for the task it is supposed to solve at test time. Our training methodology poses little restrictions on the task to learn, and works for any architecture which predicts key point heat maps, and descriptors for key point locations.},
	url = {http://arxiv.org/abs/1912.00623},
	author = {Bhowmik, Aritra and Gumhold, Stefan and Carsten Rother and Brachmann, Eric}
}
@conference {Hodan2018,
	title = {BOP: Benchmark for 6D object pose estimation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11214 LNCS},
	year = {2018},
	month = {aug},
	pages = {19{\textendash}35},
	abstract = {We propose a benchmark for 6D pose estimation of a rigid object from a single RGB-D input image. The training data consists of a texture-mapped 3D object model or images of the object in known 6D poses. The benchmark comprises of: (i) eight datasets in a unified format that cover different practical scenarios, including two new datasets focusing on varying lighting conditions, (ii) an evaluation methodology with a pose-error function that deals with pose ambiguities, (iii) a comprehensive evaluation of 15 diverse recent methods that captures the status quo of the field, and (iv) an online evaluation system that is open for continuous submission of new results. The evaluation shows that methods based on point-pair features currently perform best, outperforming template matching methods, learning-based methods and methods based on 3D local features. The project website is available at bop.felk.cvut.cz.},
	isbn = {9783030012489},
	issn = {16113349},
	doi = {10.1007/978-3-030-01249-6_2},
	url = {http://arxiv.org/abs/1808.08319},
	author = {Hoda{\v n}, Tom{\'a}{\v s} and Michel, Frank and Brachmann, Eric and Kehl, Wadim and Buch, Anders Glent and Kraft, Dirk and Drost, Bertram and Vidal, Joel and Ihrke, Stephan and Zabulis, Xenophon and Sahin, Caner and Manhardt, Fabian and Tombari, Federico and Kim, Tae Kyun and Matas, Ji{\v r}{\'\i} and Carsten Rother}
}
@conference {Brachmann2018,
	title = {Learning Less is More - 6D Camera Localization via 3D Surface Regression},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2018},
	month = {nov},
	pages = {4654{\textendash}4662},
	abstract = {Popular research areas like autonomous driving and augmented reality have renewed the interest in image-based camera localization. In this work, we address the task of predicting the 6D camera pose from a single RGB image in a given 3D environment. With the advent of neural networks, previous works have either learned the entire camera localization process, or multiple components of a camera localization pipeline. Our key contribution is to demonstrate and explain that learning a single component of this pipeline is sufficient. This component is a fully convolutional neural network for densely regressing so-called scene coordinates, defining the correspondence between the input image and the 3D scene space. The neural network is prepended to a new end-to-end trainable pipeline. Our system is efficient, highly accurate, robust in training, and exhibits outstanding generalization capabilities. It exceeds state-of-the-art consistently on indoor and outdoor datasets. Interestingly, our approach surpasses existing techniques even without utilizing a 3D model of the scene during training, since the network is able to discover 3D scene geometry automatically, solely from single-view constraints.},
	isbn = {9781538664209},
	issn = {10636919},
	doi = {10.1109/CVPR.2018.00489},
	url = {http://arxiv.org/abs/1711.10228},
	author = {Brachmann, Eric and Carsten Rother}
}
@conference {Brachmann2017,
	title = {DSAC - Differentiable RANSAC for camera localization},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {nov},
	pages = {2492{\textendash}2500},
	abstract = {RANSAC is an important algorithm in robust optimization and a central building block for many computer vision applications. In recent years, traditionally hand-crafted pipelines have been replaced by deep learning pipelines, which can be trained in an end-to-end fashion. However, RANSAC has so far not been used as part of such deep learning pipelines, because its hypothesis selection procedure is non-differentiable. In this work, we present two different ways to overcome this limitation. The most promising approach is inspired by reinforcement learning, namely to replace the deterministic hypothesis selection by a probabilistic selection for which we can derive the expected loss w.r.t. to all learnable parameters. We call this approach DSAC, the differentiable counterpart of RANSAC. We apply DSAC to the problem of camera localization, where deep learning has so far failed to improve on traditional approaches. We demonstrate that by directly minimizing the expected loss of the output camera poses, robustly estimated by RANSAC, we achieve an increase in accuracy. In the future, any deep learning pipeline can use DSAC as a robust optimization component.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.267},
	url = {http://arxiv.org/abs/1611.05705},
	author = {Brachmann, Eric and Krull, Alexander and Nowozin, Sebastian and Shotton, Jamie and Michel, Frank and Gumhold, Stefan and Carsten Rother}
}
@conference {Michel2017,
	title = {Global hypothesis generation for 6D object pose estimation},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {dec},
	pages = {115{\textendash}124},
	abstract = {This paper addresses the task of estimating the 6D pose of a known 3D object from a single RGB-D image. Most modern approaches solve this task in three steps: i) Compute local features; ii) Generate a pool of pose-hypotheses; iii) Select and refine a pose from the pool. This work focuses on the second step. While all existing approaches generate the hypotheses pool via local reasoning, e.g. RANSAC or Hough-voting, we are the first to show that global reasoning is beneficial at this stage. In particular, we formulate a novel fully-connected Conditional Random Field (CRF) that outputs a very small number of pose-hypotheses. Despite the potential functions of the CRF being non-Gaussian, we give a new and efficient two-step optimization procedure, with some guarantees for optimality. We utilize our global hypotheses generation procedure to produce results that exceed state-of-the-art for the challenging "Occluded Object Dataset".},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.20},
	url = {http://arxiv.org/abs/1612.02287},
	author = {Michel, Frank and Kirillov, Alexander and Brachmann, Eric and Krull, Alexander and Gumhold, Stefan and Savchynskyy, Bogdan and Carsten Rother}
}
@conference {Krull2017,
	title = {PoseAgent: Budget-constrained 6D object pose estimation via reinforcement learning},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {dec},
	pages = {2566{\textendash}2574},
	abstract = {State-of-the-art computer vision algorithms often achieve efficiency by making discrete choices about which hypotheses to explore next. This allows allocation of computational resources to promising candidates, however, such decisions are non-differentiable. As a result, these algorithms are hard to train in an end-to-end fashion. In this work we propose to learn an efficient algorithm for the task of 6D object pose estimation. Our system optimizes the parameters of an existing state-of-the art pose estimation system using reinforcement learning, where the pose estimation system now becomes the stochastic policy, parametrized by a CNN. Additionally, we present an efficient training algorithm that dramatically reduces computation time. We show empirically that our learned pose estimation procedure makes better use of limited resources and improves upon the state-of-the-art on a challenging dataset. Our approach enables differentiable end-to-end training of complex algorithmic pipelines and learns to make optimal use of a given computational budget.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.275},
	url = {http://arxiv.org/abs/1612.03779},
	author = {Krull, Alexander and Brachmann, Eric and Nowozin, Sebastian and Michel, Frank and Shotton, Jamie and Carsten Rother}
}
@booklet {Massiceti,
	title = {Random Forests versus Neural Networks - What{\textquoteright}s best for camera location},
	year = {2017},
	abstract = {This work addresses the task of camera localiza-tion in a known 3D scene given a single input RGB image. State-of-the-art approaches accomplish this in two steps: firstly, regressing for every pixel in the image its 3D scene coordinate and subsequently, using these coordinates to estimate the final 6D camera pose via RANSAC. To solve the first step, Random Forests (RFs) are typically used. On the other hand, Neural Networks (NNs) reign in many dense regression tasks, but are not test-time efficient. We ask the question: which of the two is best for camera localization? To address this, we make two method contributions: (1) a test-time efficient NN architecture which we term a ForestNet that is derived and initialized from a RF, and (2) a new fully-differentiable robust averaging technique for regression ensembles which can be trained end-to-end with a NN. Our experimental findings show that for scene coordinate regression, traditional NN architectures are superior to test-time efficient RFs and ForestNets, however, this does not translate to final 6D camera pose accuracy where RFs and ForestNets perform slightly better. To summarize, our best method, a ForestNet with a robust average, which has an equivalent fast and lightweight RF, improves over the state-of-the-art for camera localization on the 7-Scenes dataset [1]. While this work focuses on scene coordinate regression for camera localization, our innovations may also be applied to other continuous regression tasks.},
	author = {Massiceti, Daniela and Krull, Alexander and Brachmann, Eric and Carsten Rother and Torr, Philip H S}
}
@conference {Brachmann2016a,
	title = {Uncertainty-Driven 6D Pose Estimation of Objects and Scenes from a Single RGB Image},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {2016-Decem},
	year = {2016},
	pages = {3364{\textendash}3372},
	abstract = {In recent years, the task of estimating the 6D pose of object instances and complete scenes, i.e. camera localization, from a single input image has received considerable attention. Consumer RGB-D cameras have made this feasible, even for difficult, texture-less objects and scenes. In this work, we show that a single RGB image is sufficient to achieve visually convincing results. Our key concept is to model and exploit the uncertainty of the system at all stages of the processing pipeline. The uncertainty comes in the form of continuous distributions over 3D object coordinates and discrete distributions over object labels. We give three technical contributions. Firstly, we develop a regularized, auto-context regression framework which iteratively reduces uncertainty in object coordinate and object label predictions. Secondly, we introduce an efficient way to marginalize object coordinate distributions over depth. This is necessary to deal with missing depth information. Thirdly, we utilize the distributions over object labels to detect multiple objects simultaneously with a fixed budget of RANSAC hypotheses. We tested our system for object pose estimation and camera localization on commonly used data sets. We see a major improvement over competing systems.},
	isbn = {9781467388504},
	issn = {10636919},
	doi = {10.1109/CVPR.2016.366},
	author = {Brachmann, Eric and Michel, Frank and Krull, Alexander and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Brachmann2016,
	title = {Uncertainty-Driven 6D Pose Estimation of Objects and Scenes from a Single RGB Image},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {2016-Decem},
	year = {2016},
	pages = {3364{\textendash}3372},
	abstract = {In recent years, the task of estimating the 6D pose of object instances and complete scenes, i.e. camera localization, from a single input image has received considerable attention. Consumer RGB-D cameras have made this feasible, even for difficult, texture-less objects and scenes. In this work, we show that a single RGB image is sufficient to achieve visually convincing results. Our key concept is to model and exploit the uncertainty of the system at all stages of the processing pipeline. The uncertainty comes in the form of continuous distributions over 3D object coordinates and discrete distributions over object labels. We give three technical contributions. Firstly, we develop a regularized, auto-context regression framework which iteratively reduces uncertainty in object coordinate and object label predictions. Secondly, we introduce an efficient way to marginalize object coordinate distributions over depth. This is necessary to deal with missing depth information. Thirdly, we utilize the distributions over object labels to detect multiple objects simultaneously with a fixed budget of RANSAC hypotheses. We tested our system for object pose estimation and camera localization on commonly used data sets. We see a major improvement over competing systems.},
	isbn = {9781467388504},
	issn = {10636919},
	doi = {10.1109/CVPR.2016.366},
	author = {Brachmann, Eric and Michel, Frank and Krull, Alexander and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Krull2015,
	title = {Learning analysis-by-synthesis for 6d pose estimation in RGB-D images},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2015 Inter},
	year = {2015},
	pages = {954{\textendash}962},
	abstract = {Analysis-by-synthesis has been a successful approach for many tasks in computer vision, such as 6D pose estimation of an object in an RGB-D image which is the topic of this work. The idea is to compare the observation with the output of a forward process, such as a rendered image of the object of interest in a particular pose. Due to occlusion or complicated sensor noise, it can be difficult to perform this comparison in a meaningful way. We propose an approach that "learns to compare", while taking these difficulties into account. This is done by describing the posterior density of a particular object pose with a convolutional neural network (CNN) that compares observed and rendered images. The network is trained with the maximum likelihood paradigm. We observe empirically that the CNN does not specialize to the geometry or appearance of specific objects. It can be used with objects of vastly different shapes and appearances, and in different backgrounds. Compared to state-of-the-art, we demonstrate a significant improvement on two different datasets which include a total of eleven objects, cluttered background, and heavy occlusion.},
	isbn = {9781467383912},
	issn = {15505499},
	doi = {10.1109/ICCV.2015.115},
	author = {Krull, Alexander and Brachmann, Eric and Michel, Frank and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Michel2015,
	title = {Pose Estimation of Kinematic Chain Instances via Object Coordinate Regression},
	year = {2015},
	pages = {181.1{\textendash}181.11},
	abstract = {In this paper, we address the problem of one shot pose estimation of articulated ob-jects from an RGB-D image. In particular, we consider object instances with the topol-ogy of a kinematic chain, i.e. assemblies of rigid parts connected by prismatic or revolute joints. This object type occurs often in daily live, for instance in the form of furniture or electronic devices. Instead of treating each object part separately we are using the rela-tionship between parts of the kinematic chain and propose a new minimal pose sampling approach. This enables us to create a pose hypothesis for a kinematic chain consist-ing of K parts by sampling K 3D-3D point correspondences. To asses the quality of our method, we gathered a large dataset containing four objects and 7000+ annotated RGB-D frames 1 . On this dataset we achieve considerably better results than a modified state-of-the-art pose estimation system for rigid objects.},
	doi = {10.5244/c.29.181},
	author = {Michel, Frank and Krull, Alexander and Brachmann, Eric and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
