@article {Krull2020,
	title = {Artificial-intelligence-driven scanning probe microscopy},
	journal = {Communications Physics},
	volume = {3},
	year = {2020},
	abstract = {Scanning probe microscopy (SPM) has revolutionized the fields of materials, nano-science, chemistry, and biology, by enabling mapping of surface properties and surface manipulation with atomic precision. However, these achievements require constant human supervision; fully automated SPM has not been accomplished yet. Here we demonstrate an artificial intelligence framework based on machine learning for autonomous SPM operation (DeepSPM). DeepSPM includes an algorithmic search of good sample regions, a convolutional neural network to assess the quality of acquired images, and a deep reinforcement learning agent to reliably condition the state of the probe. DeepSPM is able to acquire and classify data continuously in multi-day scanning tunneling microscopy experiments, managing the probe quality in response to varying experimental conditions. Our approach paves the way for advanced methods hardly feasible by human operation (e.g., large dataset acquisition and SPM-based nanolithography). DeepSPM can be generalized to most SPM techniques, with the source code publicly available.},
	issn = {23993650},
	doi = {10.1038/s42005-020-0317-3},
	author = {Krull, A. and Hirsch, P. and Rother, C. and Schiffrin, A. and Krull, C.}
}
@conference {Kluger2020,
	title = {CONSAC: Robust Multi-Model Fitting by Conditional Sample Consensus},
	booktitle = {CVPR 2020},
	year = {2020},
	month = {01/2020},
	abstract = {We present a robust estimator for fitting multiple parametric models of the same form to noisy measurements. Applications include finding multiple vanishing points in man-made scenes, fitting planes to architectural imagery, or estimating multiple rigid motions within the same sequence. In contrast to previous works, which resorted to hand-crafted search strategies for multiple model detection, we learn the search strategy from data. A neural network conditioned on previously detected models guides a RANSAC estimator to different subsets of all measurements, thereby finding model instances one after another. We train our method supervised as well as self-supervised. For supervised training of the search strategy, we contribute a new dataset for vanishing point estimation. Leveraging this dataset, the proposed algorithm is superior with respect to other robust estimators as well as to designated vanishing point estimation algorithms. For self-supervised learning of the search, we evaluate the proposed algorithm on multi-homography estimation and demonstrate an accuracy that is superior to state-of-the-art methods.},
	url = {http://arxiv.org/abs/2001.02643},
	author = {Kluger, Florian and Brachmann, Eric and Ackermann, Hanno and Carsten Rother and Yang, Michael Ying and Rosenhahn, Bodo}
}
@article {Ardizzone2020,
	title = {Exact Information Bottleneck with Invertible Neural Networks: Getting the Best of Discriminative and Generative Modeling},
	year = {2020},
	month = {jan},
	abstract = {Generative models are more informative about underlying phenomena than discriminative ones and offer superior uncertainty quantification and out-of-distribution robustness. However, these advantages often come at the expense of reduced classification accuracy. The Information Bottleneck objective (IB) formulates this trade-off in a clean information-theoretic way, but its practical application is hampered by a lack of accurate high-dimensional estimators of mutual information (MI), its main constituent. To overcome this limitation, we develop the theory and methodology of IB-INNs, which optimize the IB objective by means of Invertible Neural Networks (INNs), without the need for approximations of MI. Our experiments show that IB-INNs allow for a precise adjustment of the generative/discriminative trade-off: They learn accurate models of the class conditional likelihoods, generalize well to unseen data and reliably detect out-of-distribution examples, while at the same time exhibiting classification accuracy close to purely discriminative feed-forward networks.},
	url = {http://arxiv.org/abs/2001.06448},
	author = {Lynton Ardizzone and Mackowiak, Radek and Carsten Rother and Ullrich K{\"o}the}
}
@proceedings {Haller2020,
	title = {A Primal-Dual Solver for Large-Scale Tracking-by-Assignment},
	year = {2020},
	abstract = {We propose a fast approximate solver for the combinatorial problem known as tracking-by-assignment, which we apply to cell tracking. The latter plays a key role in discovery in many life sciences, especially in cell and developmental biology. So far, in the most general setting this problem was addressed by off-the-shelf solvers like Gurobi, whose run time and memory requirements rapidly grow with the size of the input. In contrast, for our method this growth is nearly linear. Our contribution consists of a new (1) de-composable compact representation of the problem; (2) dual block-coordinate ascent method for optimizing the decomposition-based dual; and (3) primal heuristics that reconstructs a feasible integer solution based on the dual information. Compared to solving the problem with Gurobi, we observe an up to 60 times speed-up, while reducing the memory footprint significantly. We demonstrate the efficacy of our method on real-world tracking problems.},
	author = {Stefan Haller and Prakash, Mangal and Hutschenreiter, Lisa and Pietzsch, Tobias and Carsten Rother and Jug, Florian and Swoboda, Paul and Savchynskyy, Bogdan}
}
@conference {Tourani2020,
	title = {Taxonomy of Dual Block-Coordinate Ascent Methods for Discrete Energy Minimization},
	booktitle = {AISTATS 2020},
	year = {2020},
	abstract = {We consider the maximum-a-posteriori inference problem in discrete graphical models and study solvers based on the dual block-coordinate ascent rule. We map all existing solvers in a single framework, allowing for a better understanding of their design principles. We theoretically show that some block-optimizing updates are sub-optimal and how to strictly improve them. On a wide range of problem instances of varying graph connec-tivity, we study the performance of existing solvers as well as new variants that can be obtained within the framework. As a result of this exploration we build a new state-of-the art solver, performing uniformly better on the whole range of test instances.},
	url = {https://gitlab.com/},
	author = {Tourani, Siddharth and Shekhovtsov, Alexander and Carsten Rother and Savchynskyy, Bogdan}
}
@techreport {Kruse2019,
	title = {Benchmarking Invertible Architectures on Inverse Problems},
	number = {i},
	year = {2019},
	abstract = {Recent work demonstrated that flow-based invert-ible neural networks are promising tools for solving ambiguous inverse problems. Following up on this, we investigate how ten invertible archi-tectures and related models fare on two intuitive, low-dimensional benchmark problems, obtaining the best results with coupling layers and simple autoencoders. We hope that our initial efforts inspire other researchers to evaluate their invertible architectures in the same setting and put forth additional benchmarks, so our evaluation may eventually grow into an official community challenge.},
	author = {Kruse, Jakob and Lynton Ardizzone and Carsten Rother and Ullrich K{\"o}the}
}
@article {Kamann2019,
	title = {Benchmarking the Robustness of Semantic Segmentation Models},
	year = {2019},
	abstract = {When designing a semantic segmentation module for a practical application, such as autonomous driving, it is crucial to understand the robustness of the module with respect to a wide range of image corruptions. While there are recent robustness studies for full-image classification, we are the first to present an exhaustive study for semantic segmentation, based on the state-of-the-art model DeepLabv3\$+\$. To increase the realism of our study, we utilize almost 200,000 images generated from Cityscapes and PASCAL VOC 2012, and we furthermore present a realistic noise model, imitating HDR camera noise. Based on the benchmark study we gain several new insights. Firstly, model robustness increases with model performance, in most cases. Secondly, some architecture properties affect robustness significantly, such as a Dense Prediction Cell which was designed to maximize performance on clean data only. Thirdly, to achieve good generalization with respect to various types of image noise, it is recommended to train DeepLabv3+ with our realistic noise model.},
	url = {http://arxiv.org/abs/1908.05005},
	author = {Kamann, Christoph and Carsten Rother}
}
@conference {Mackowiak2019,
	title = {CEREALS - Cost-Effective REgion-based Active Learning for Semantic Segmentation},
	booktitle = {British Machine Vision Conference 2018, BMVC 2018},
	year = {2019},
	abstract = {State of the art methods for semantic image segmentation are trained in a supervised fashion using a large corpus of fully labeled training images. However, gathering such a corpus is expensive, due to human annotation effort, in contrast to gathering unlabeled data. We propose an active learning-based strategy, called CEREALS, in which a human only has to hand-label a few, automatically selected, regions within an unlabeled image corpus. This minimizes human annotation effort while maximizing the performance of a semantic image segmentation method. The automatic selection procedure is achieved by: a) using a suitable information measure combined with an estimate about human annotation effort, which is inferred from a learned cost model, and b) exploiting the spatial coherency of an image. The performance of CEREALS is demonstrated on Cityscapes, where we are able to reduce the annotation effort to 17\%, while keeping 95\% of the mean Intersection over Union (mIoU) of a model that was trained with the fully annotated training set of Cityscapes.},
	author = {Mackowiak, Radek and Lenz, Philip and Ghori, Omair and Ferran Diego and Lange, Oliver and Carsten Rother}
}
@conference {Li2019,
	title = {Deep Object Co-segmentation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11363 LNCS},
	year = {2019},
	pages = {638{\textendash}653},
	abstract = {This work presents a deep object co-segmentation (DOCS) approach for segmenting common objects of the same class within a pair of images. This means that the method learns to ignore common, or uncommon, background stuff and focuses on common objects. If multiple object classes are presented in the image pair, they are jointly extracted as foreground. To address this task, we propose a CNN-based Siamese encoder-decoder architecture. The encoder extracts high-level semantic features of the foreground objects, a mutual correlation layer detects the common objects, and finally, the decoder generates the output foreground masks for each image. To train our model, we compile a large object co-segmentation dataset consisting of image pairs from the PASCAL dataset with common objects masks. We evaluate our approach on commonly used datasets for co-segmentation tasks and observe that our approach consistently outperforms competing methods, for both seen and unseen object classes.},
	isbn = {9783030208929},
	issn = {16113349},
	doi = {10.1007/978-3-030-20893-6_40},
	author = {Weihao Li and Omid Hosseini Jafari and Carsten Rother}
}
@conference {Brachmann2019,
	title = {Expert sample consensus applied to camera re-localization},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2019-Octob},
	year = {2019},
	month = {aug},
	pages = {7524{\textendash}7533},
	abstract = {Fitting model parameters to a set of noisy data points is a common problem in computer vision. In this work, we fit the 6D camera pose to a set of noisy correspondences between the 2D input image and a known 3D environment. We estimate these correspondences from the image using a neural network. Since the correspondences often contain outliers, we utilize a robust estimator such as Random Sample Consensus (RANSAC) or Differentiable RANSAC (DSAC) to fit the pose parameters. When the problem domain, e.g. the space of all 2D-3D correspondences, is large or ambiguous, a single network does not cover the domain well. Mixture of Experts (MoE) is a popular strategy to divide a problem domain among an ensemble of specialized networks, so called experts, where a gating network decides which expert is responsible for a given input. In this work, we introduce Expert Sample Consensus (ESAC), which integrates DSAC in a MoE. Our main technical contribution is an efficient method to train ESAC jointly and end-to-end. We demonstrate experimentally that ESAC handles two real-world problems better than competing methods, i.e. scalability and ambiguity. We apply ESAC to fitting simple geometric models to synthetic images, and to camera re-localization for difficult, real datasets.},
	isbn = {9781728148038},
	issn = {15505499},
	doi = {10.1109/ICCV.2019.00762},
	url = {http://arxiv.org/abs/1908.02484},
	author = {Brachmann, Eric and Carsten Rother}
}
@conference {AbuAlhaija2019,
	title = {Geometric Image Synthesis},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11366 LNCS},
	year = {2019},
	pages = {85{\textendash}100},
	abstract = {The task of generating natural images from 3D scenes has been a long standing goal in computer graphics. On the other hand, recent developments in deep neural networks allow for trainable models that can produce natural-looking images with little or no knowledge about the scene structure. While the generated images often consist of realistic looking local patterns, the overall structure of the generated images is often inconsistent. In this work we propose a trainable, geometry-aware image generation method that leverages various types of scene information, including geometry and segmentation, to create realistic looking natural images that match the desired scene structure. Our geometrically-consistent image synthesis method is a deep neural network, called Geometry to Image Synthesis (GIS) framework, which retains the advantages of a trainable method, e.g., differentiability and adaptiveness, but, at the same time, makes a step towards the generalizability, control and quality output of modern graphics rendering engines. We utilize the GIS framework to insert vehicles in outdoor driving scenes, as well as to generate novel views of objects from the Linemod dataset. We qualitatively show that our network is able to generalize beyond the training set to novel scene geometries, object shapes and segmentations. Furthermore, we quantitatively show that the GIS framework can be used to synthesize large amounts of training data which proves beneficial for training instance segmentation models.},
	isbn = {9783030208752},
	issn = {16113349},
	doi = {10.1007/978-3-030-20876-9_6},
	url = {https://youtu.be/W2tFCz9xJoU},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Geiger, Andreas and Carsten Rother}
}
@article {Ardizzone2019,
	title = {Guided Image Generation with Conditional Invertible Neural Networks},
	year = {2019},
	abstract = {In this work, we address the task of natural image generation guided by a conditioning input. We introduce a new architecture called conditional invertible neural network (cINN). The cINN combines the purely generative INN model with an unconstrained feed-forward network, which efficiently preprocesses the conditioning input into useful features. All parameters of the cINN are jointly optimized with a stable, maximum likelihood-based training procedure. By construction, the cINN does not experience mode collapse and generates diverse samples, in contrast to e.g. cGANs. At the same time our model produces sharp images since no reconstruction loss is required, in contrast to e.g. VAEs. We demonstrate these properties for the tasks of MNIST digit generation and image colorization. Furthermore, we take advantage of our bi-directional cINN architecture to explore and manipulate emergent properties of the latent space, such as changing the image style in an intuitive way.},
	url = {http://arxiv.org/abs/1907.02392},
	author = {Lynton Ardizzone and Carsten L{\"u}th and Kruse, Jakob and Carsten Rother and Ullrich K{\"o}the}
}
@article {Ardizzone2019a,
	title = {Guided Image Generation with Conditional Invertible Neural Networks},
	year = {2019},
	abstract = {In this work, we address the task of natural image generation guided by a conditioning input. We introduce a new architecture called conditional invertible neural network (cINN). The cINN combines the purely generative INN model with an unconstrained feed-forward network, which efficiently preprocesses the conditioning input into useful features. All parameters of the cINN are jointly optimized with a stable, maximum likelihood-based training procedure. By construction, the cINN does not experience mode collapse and generates diverse samples, in contrast to e.g. cGANs. At the same time our model produces sharp images since no reconstruction loss is required, in contrast to e.g. VAEs. We demonstrate these properties for the tasks of MNIST digit generation and image colorization. Furthermore, we take advantage of our bi-directional cINN architecture to explore and manipulate emergent properties of the latent space, such as changing the image style in an intuitive way.},
	url = {http://arxiv.org/abs/1907.02392},
	author = {Lynton Ardizzone and Carsten L{\"u}th and Kruse, Jakob and Carsten Rother and Ullrich K{\"o}the}
}
@conference {HosseiniJafari2019,
	title = {iPose: Instance-Aware 6D Pose Estimation of Partly Occluded Objects},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11363 LNCS},
	year = {2019},
	pages = {477{\textendash}492},
	abstract = {We address the task of 6D pose estimation of known rigid objects from single input images in scenarios where the objects are partly occluded. Recent RGB-D-based methods are robust to moderate degrees of occlusion. For RGB inputs, no previous method works well for partly occluded objects. Our main contribution is to present the first deep learning-based system that estimates accurate poses for partly occluded objects from RGB-D and RGB input. We achieve this with a new instance-aware pipeline that decomposes 6D object pose estimation into a sequence of simpler steps, where each step removes specific aspects of the problem. The first step localizes all known objects in the image using an instance segmentation network, and hence eliminates surrounding clutter and occluders. The second step densely maps pixels to 3D object surface positions, so called object coordinates, using an encoder-decoder network, and hence eliminates object appearance. The third, and final, step predicts the 6D pose using geometric optimization. We demonstrate that we significantly outperform the state-of-the-art for pose estimation of partly occluded objects for both RGB and RGB-D input.},
	isbn = {9783030208929},
	issn = {16113349},
	doi = {10.1007/978-3-030-20893-6_30},
	author = {Omid Hosseini Jafari and Mustikovela, Siva Karthik and Pertsch, Karl and Brachmann, Eric and Carsten Rother}
}
@conference {Leistner2019,
	title = {Learning to Think Outside the Box: Wide-Baseline Light Field Depth Estimation with EPI-Shift},
	booktitle = {Proceedings - 2019 International Conference on 3D Vision, 3DV 2019},
	year = {2019},
	month = {sep},
	pages = {249{\textendash}257},
	abstract = {We propose a method for depth estimation from light field data, based on a fully convolutional neural network architecture. Our goal is to design a pipeline which achieves highly accurate results for small-and wide-baseline light fields. Since light field training data is scarce, all learning-based approaches use a small receptive field and operate on small disparity ranges. In order to work with wide-baseline light fields, we introduce the idea of EPI-Shift: To virtually shift the light field stack which enables to retain a small receptive field, independent of the disparity range. In this way, our approach {\textquoteright}learns to think outside the box of the receptive field". Our network performs joint classification of integer disparities and regression of disparity-offsets. A U-Net component provides excellent long-range smoothing. EPI-Shift considerably outperforms the state-of-the-art learning-based approaches and is on par with hand-crafted methods. We demonstrate this on a publicly available, synthetic, small-baseline benchmark and on large-baseline real-world recordings.},
	keywords = {Computer vision, deep learning, depth estimation, light fields, Stereo},
	isbn = {9781728131313},
	doi = {10.1109/3DV.2019.00036},
	url = {http://arxiv.org/abs/1909.09059 http://dx.doi.org/10.1109/3DV.2019.00036},
	author = {Titus Leistner and Schilling, Hendrik and Mackowiak, Radek and Gumhold, Stefan and Carsten Rother}
}
@techreport {Li,
	title = {Localizing Common Objects Using Common Component Activation Map},
	year = {2019},
	abstract = {In this work, we propose an approach to localize common objects from novel object categories in a set of images. We solve this problem using a new common component activation map (CCAM) in which we treat the class-specific activation maps (CAM) as components to discover the common components in the image set. We show that our approach can generalize on novel object categories in our experiments .},
	author = {Weihao Li and Omid Hosseini Jafari and Carsten Rother}
}
@conference {Brachmann2019a,
	title = {Neural-guided RANSAC: Learning where to sample model hypotheses},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2019-Octob},
	year = {2019},
	month = {may},
	pages = {4321{\textendash}4330},
	abstract = {We present Neural-Guided RANSAC (NG-RANSAC), an extension to the classic RANSAC algorithm from robust optimization. NG-RANSAC uses prior information to improve model hypothesis search, increasing the chance of finding outlier-free minimal sets. Previous works use heuristic side-information like hand-crafted descriptor distance to guide hypothesis search. In contrast, we learn hypothesis search in a principled fashion that lets us optimize an arbitrary task loss during training, leading to large improvements on classic computer vision tasks. We present two further extensions to NG-RANSAC. Firstly, using the inlier count itself as training signal allows us to train neural guidance in a self-supervised fashion. Secondly, we combine neural guidance with differentiable RANSAC to build neural networks which focus on certain parts of the input data and make the output predictions as good as possible. We evaluate NG-RANSAC on a wide array of computer vision tasks, namely estimation of epipolar geometry, horizon line estimation and camera re-localization. We achieve superior or competitive results compared to state-of-the-art robust estimators, including very recent, learned ones.},
	isbn = {9781728148038},
	issn = {15505499},
	doi = {10.1109/ICCV.2019.00442},
	url = {http://arxiv.org/abs/1905.04132},
	author = {Brachmann, Eric and Carsten Rother}
}
@conference {Adler2019,
	title = {Out of Distribution Detection for Intra-operative Functional Imaging},
	booktitle = {MICCAI UNSURE Workshop 2019},
	volume = {11840 LNCS},
	year = {2019},
	pages = {75{\textendash}82},
	abstract = {Multispectral optical imaging is becoming a key tool in the operating room. Recent research has shown that machine learning algorithms can be used to convert pixel-wise reflectance measurements to tissue parameters, such as oxygenation. However, the accuracy of these algorithms can only be guaranteed if the spectra acquired during surgery match the ones seen during training. It is therefore of great interest to detect so-called out of distribution (OoD) spectra to prevent the algorithm from presenting spurious results. In this paper we present an information theory based approach to OoD detection based on the widely applicable information criterion (WAIC). Our work builds upon recent methodology related to invertible neural networks (INN). Specifically, we make use of an ensemble of INNs as we need their tractable Jacobians in order to compute the WAIC. Comprehensive experiments with in silico, and in vivo multispectral imaging data indicate that our approach is well-suited for OoD detection. Our method could thus be an important step towards reliable functional imaging in the operating room.},
	issn = {16113349},
	doi = {10.1007/978-3-030-32689-0_8},
	author = {Adler, Tim J and Ayala, Leonardo and Lynton Ardizzone and Kenngott, Hannes G and Vemuri, Anant and M{\"u}ller-Stich, Beat P and Carsten Rother and Ullrich K{\"o}the and Maier-Hein, Lena}
}
@article {Bhowmik2019,
	title = {Reinforced Feature Points: Optimizing Feature Detection and Description for a High-Level Task},
	year = {2019},
	month = {dec},
	abstract = {We address a core problem of computer vision: Detection and description of 2D feature points for image matching. For a long time, hand-crafted designs, like the seminal SIFT algorithm, were unsurpassed in accuracy and efficiency. Recently, learned feature detectors emerged that implement detection and description using neural networks. Training these networks usually resorts to optimizing low-level matching scores, often pre-defining sets of image patches which should or should not match, or which should or should not contain key points. Unfortunately, increased accuracy for these low-level matching scores does not necessarily translate to better performance in high-level vision tasks. We propose a new training methodology which embeds the feature detector in a complete vision pipeline, and where the learnable parameters are trained in an end-to-end fashion. We overcome the discrete nature of key point selection and descriptor matching using principles from reinforcement learning. As an example, we address the task of relative pose estimation between a pair of images. We demonstrate that the accuracy of a state-of-the-art learning-based feature detector can be increased when trained for the task it is supposed to solve at test time. Our training methodology poses little restrictions on the task to learn, and works for any architecture which predicts key point heat maps, and descriptors for key point locations.},
	url = {http://arxiv.org/abs/1912.00623},
	author = {Bhowmik, Aritra and Gumhold, Stefan and Carsten Rother and Brachmann, Eric}
}
@article {Alhaija2018,
	title = {Augmented Reality Meets Computer Vision},
	journal = {International Journal of Computer Vision},
	volume = {In press},
	year = {2018},
	pages = {1{\textendash}13},
	abstract = {The success of deep learning in computer vision is based on availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment real images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance and a large number of complex object arrangements. In contrast to modeling complete 3D environments, our augmentation approach requires only a few user interactions in combination with 3D shapes of the target object. Through extensive experimentation, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of our approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenes. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on Cityscapes dataset. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amount of annotated real data.},
	keywords = {autonomous driving, data augmenta-, instance segmentation, synthetic training data, tion},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@article {AbuAlhaija2018,
	title = {Augmented Reality Meets Computer Vision: Efficient Data Generation for Urban Driving Scenes},
	journal = {International Journal of Computer Vision},
	volume = {126},
	number = {9},
	year = {2018},
	month = {aug},
	pages = {961{\textendash}972},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D models of the target object category. Leveraging our approach, we introduce a novel dataset of augmented urban driving scenes with 360 degree images that are used as environment maps to create realistic lighting and reflections on rendered objects. We analyze the significance of realistic object placement by comparing manual placement by humans to automatic methods based on semantic scene analysis. This allows us to create composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. Through an extensive set of experiments, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of the proposed approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenarios. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on the Cityscapes dataset. Our experiments demonstrate that the models trained on augmented imagery generalize better than those trained on fully synthetic data or models trained on limited amounts of annotated real data.},
	keywords = {autonomous driving, Data augmentation, instance segmentation, Object detection, synthetic training data},
	issn = {15731405},
	doi = {10.1007/s11263-018-1070-x},
	url = {http://arxiv.org/abs/1708.01566},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@article {6268,
	title = {Augmented Reality Meets Computer Vision Efficient Data Generation for Urban Driving Scenes},
	journal = {IJCV},
	year = {2018},
	pages = {1-12},
	doi = {10.1007/s11263-018-1070-x},
	author = {Hassan Abu Alhaija and Mustikovela, S K and Mescheder, A and Geiger, C and Carsten Rother}
}
@conference {Hodan2018,
	title = {BOP: Benchmark for 6D object pose estimation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11214 LNCS},
	year = {2018},
	month = {aug},
	pages = {19{\textendash}35},
	abstract = {We propose a benchmark for 6D pose estimation of a rigid object from a single RGB-D input image. The training data consists of a texture-mapped 3D object model or images of the object in known 6D poses. The benchmark comprises of: (i) eight datasets in a unified format that cover different practical scenarios, including two new datasets focusing on varying lighting conditions, (ii) an evaluation methodology with a pose-error function that deals with pose ambiguities, (iii) a comprehensive evaluation of 15 diverse recent methods that captures the status quo of the field, and (iv) an online evaluation system that is open for continuous submission of new results. The evaluation shows that methods based on point-pair features currently perform best, outperforming template matching methods, learning-based methods and methods based on 3D local features. The project website is available at bop.felk.cvut.cz.},
	isbn = {9783030012489},
	issn = {16113349},
	doi = {10.1007/978-3-030-01249-6_2},
	url = {http://arxiv.org/abs/1808.08319},
	author = {Hoda{\v n}, Tom{\'a}{\v s} and Michel, Frank and Brachmann, Eric and Kehl, Wadim and Buch, Anders Glent and Kraft, Dirk and Drost, Bertram and Vidal, Joel and Ihrke, Stephan and Zabulis, Xenophon and Sahin, Caner and Manhardt, Fabian and Tombari, Federico and Kim, Tae Kyun and Matas, Ji{\v r}{\'\i} and Carsten Rother}
}
@article {Arnab2018,
	title = {Conditional Random Fields Meet Deep Neural Networks for Semantic Segmentation},
	journal = {Cvpr},
	volume = {XX},
	number = {Xx},
	year = {2018},
	pages = {1{\textendash}15},
	abstract = {{\textemdash}Semantic Segmentation is the task of labelling every pixel in an image with a pre-defined object category. It has numer-ous applications in scenarios where the detailed understanding of an image is required, such as in autonomous vehicles and medical diagnosis. This problem has traditionally been solved with probabilistic models known as Conditional Random Fields (CRFs) due to their ability to model the relationships between the pixels being predicted. However, Deep Neural Networks (DNNs) have recently been shown to excel at a wide range of computer vision problems due to their ability to learn rich feature representations automatically from data, as opposed to traditional hand-crafted features. The idea of combining CRFs and DNNs have achieved state-of-the-art results in a number of domains. We review the literature on combining the modelling power of CRFs with the representation-learning ability of DNNs, ranging from early work that combines these two techniques as independent stages of a common pipeline to recent approaches that embed inference of probabilistic models directly in the neural network itself. Finally, we summarise future research directions.},
	keywords = {conditional random fields, deep learning, seman-},
	url = {http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.308.8889\&rep=rep1\&type=pdf\%0Ahttp://dx.doi.org/10.1109/CVPR.2012.6248050},
	author = {Arnab, Anurag and Zheng, Shuai and Jayasumana, Sadeep and Romera-paredes, Bernardino and Kirillov, Alexander and Savchynskyy, Bogdan and Carsten Rother and Kahl, Fredrik and Torr, Philip}
}
@proceedings {6309,
	title = {Geometric Image Synthesis},
	year = {2018},
	author = {Hassan Abu Alhaija and Mustikovela, S K and Geiger, A and Carsten Rother}
}
@proceedings {6310,
	title = {iPose: Instance-Aware 6D Pose Estimation of Partly Occluded Objects},
	year = {2018},
	author = {Omid Hosseini Jafari and Mustikovela, S K and Pertsch, K and E Brachmann and Carsten Rother}
}
@conference {Brachmann2018,
	title = {Learning Less is More - 6D Camera Localization via 3D Surface Regression},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2018},
	month = {nov},
	pages = {4654{\textendash}4662},
	abstract = {Popular research areas like autonomous driving and augmented reality have renewed the interest in image-based camera localization. In this work, we address the task of predicting the 6D camera pose from a single RGB image in a given 3D environment. With the advent of neural networks, previous works have either learned the entire camera localization process, or multiple components of a camera localization pipeline. Our key contribution is to demonstrate and explain that learning a single component of this pipeline is sufficient. This component is a fully convolutional neural network for densely regressing so-called scene coordinates, defining the correspondence between the input image and the 3D scene space. The neural network is prepended to a new end-to-end trainable pipeline. Our system is efficient, highly accurate, robust in training, and exhibits outstanding generalization capabilities. It exceeds state-of-the-art consistently on indoor and outdoor datasets. Interestingly, our approach surpasses existing techniques even without utilizing a 3D model of the scene during training, since the network is able to discover 3D scene geometry automatically, solely from single-view constraints.},
	isbn = {9781538664209},
	issn = {10636919},
	doi = {10.1109/CVPR.2018.00489},
	url = {http://arxiv.org/abs/1711.10228},
	author = {Brachmann, Eric and Carsten Rother}
}
@conference {Tourani2018,
	title = {MPLP++: Fast, Parallel Dual Block-Coordinate Ascent for Dense Graphical Models},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11208 LNCS},
	year = {2018},
	pages = {264{\textendash}281},
	abstract = {Dense, discrete Graphical Models with pairwise potentials are a powerful class of models which are employed in state-of-the-art computer vision and bio-imaging applications. This work introduces a new MAP-solver, based on the popular Dual Block-Coordinate Ascent principle. Surprisingly, by making a small change to a low-performing solver, the Max Product Linear Programming (MPLP) algorithm [7], we derive the new solver MPLP++ that significantly outperforms all existing solvers by a large margin, including the state-of-the-art solver Tree-Reweighted Sequential (TRW-S) message-passing algorithm [17]. Additionally, our solver is highly parallel, in contrast to TRW-S, which gives a further boost in performance with the proposed GPU and multi-thread CPU implementations. We verify the superiority of our algorithm on dense problems from publicly available benchmarks as well as a new benchmark for 6D Object Pose estimation. We also provide an ablation study with respect to graph density.},
	keywords = {Block-Coordinate-Ascent, graphical models, Message passing algorithms},
	isbn = {9783030012243},
	issn = {16113349},
	doi = {10.1007/978-3-030-01225-0_16},
	author = {Tourani, Siddharth and Shekhovtsov, Alexander and Carsten Rother and Savchynskyy, Bogdan}
}
@conference {Schilling2018,
	title = {Trust your Model: Light Field Depth Estimation with Inline Occlusion Handling},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2018},
	pages = {4530{\textendash}4538},
	abstract = {We address the problem of depth estimation from light-field images. Our main contribution is a new way to handle occlusions which improves general accuracy and quality of object borders. In contrast to all prior work we work with a model which directly incorporates both depth and occlusion, using a local optimization scheme based on the PatchMatch algorithm. The key benefit of this joint approach is that we utilize all available data, and not erroneously discard valuable information in pre-processing steps. We see the benefit of our approach not only at improved object boundaries, but also at smooth surface reconstruction, where we outperform even methods which focus on good surface regularization. We have evaluated our method on a public light-field dataset, where we achieve state-of-the-art results in nine out of twelve error metrics, with a close tie for the remaining three.},
	isbn = {9781538664209},
	issn = {10636919},
	doi = {10.1109/CVPR.2018.00476},
	author = {Schilling, Hendrik and Maximilian Diebold and Carsten Rother and Bernd J{\"a}hne}
}
@proceedings {6267,
	title = {Trust your Model: Light Field Depth Estimation with inline Occlusion Handling},
	year = {2018},
	doi = {10.1109/CVPR.2018.00476},
	author = {Hendrik Schilling and Maximilian Diebold and Carsten Rother and Bernd J{\"a}hne}
}
@conference {Jafari2017,
	title = {Analyzing modular CNN architectures for joint depth prediction and semantic segmentation},
	booktitle = {Proceedings - IEEE International Conference on Robotics and Automation},
	year = {2017},
	month = {feb},
	pages = {4620{\textendash}4627},
	abstract = {This paper addresses the task of designing a modular neural network architecture that jointly solves different tasks. As an example we use the tasks of depth estimation and semantic segmentation given a single RGB image. The main focus of this work is to analyze the cross-modality influence between depth and semantic prediction maps on their joint refinement. While most of the previous works solely focus on measuring improvements in accuracy, we propose a way to quantify the cross-modality influence. We show that there is a relationship between final accuracy and cross-modality influence, although not a simple linear one. Hence a larger cross-modality influence does not necessarily translate into an improved accuracy. We find that a beneficial balance between the cross-modality influences can be achieved by network architecture and conjecture that this relationship can be utilized to understand different network design choices. Towards this end we propose a Convolutional Neural Network (CNN) architecture that fuses the state-of-the-art results for depth estimation and semantic labeling. By balancing the cross-modality influences between depth and semantic prediction, we achieve improved results for both tasks using the NYU-Depth v2 benchmark.},
	isbn = {9781509046331},
	issn = {10504729},
	doi = {10.1109/ICRA.2017.7989537},
	url = {http://arxiv.org/abs/1702.08009 http://dx.doi.org/10.1109/ICRA.2017.7989537},
	author = {Omid Hosseini Jafari and Groth, Oliver and Kirillov, Alexander and Yang, Michael Ying and Carsten Rother}
}
@conference {Alhaija2017,
	title = {Augmented reality meets deep learning for car instance segmentation in urban scenes},
	booktitle = {British Machine Vision Conference 2017, BMVC 2017},
	year = {2017},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D shapes of the target object category. We demonstrate the utility of the proposed approach for training a state-of-the-art high-capacity deep model for semantic instance segmentation. In particular, we consider the task of segmenting car instances on the KITTI dataset which we have annotated with pixel-accurate ground truth. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amounts of annotated real data.},
	isbn = {190172560X},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@conference {Behl2017a,
	title = {Bounding Boxes, Segmentations and Object Coordinates: How Important is Recognition for 3D Scene Flow Estimation in Autonomous Driving Scenarios?},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2017-Octob},
	year = {2017},
	pages = {2593{\textendash}2602},
	abstract = {Existing methods for 3D scene flow estimation often fail in the presence of large displacement or local ambiguities, e.g., at texture-less or reflective surfaces. However, these challenges are omnipresent in dynamic road scenes, which is the focus of this work. Our main contribution is to overcome these 3D motion estimation problems by exploiting recognition. In particular, we investigate the importance of recognition granularity, from coarse 2D bounding box estimates over 2D instance segmentations to fine-grained 3D object part predictions. We compute these cues using CNNs trained on a newly annotated dataset of stereo images and integrate them into a CRF-based model for robust 3D scene flow estimation - an approach we term Instance Scene Flow. We analyze the importance of each recognition cue in an ablation study and observe that the instance segmentation cue is by far strongest, in our setting. We demonstrate the effectiveness of our method on the challenging KITTI 2015 scene flow benchmark where we achieve state-of-the-art performance at the time of submission.},
	isbn = {9781538610329},
	issn = {15505499},
	doi = {10.1109/ICCV.2017.281},
	author = {Behl, Aseem and Omid Hosseini Jafari and Mustikovela, Siva Karthik and Hassan Abu Alhaija and Carsten Rother and Geiger, Andreas}
}
@conference {Behl2017,
	title = {Bounding Boxes, Segmentations and Object Coordinates: How Important is Recognition for 3D Scene Flow Estimation in Autonomous Driving Scenarios?},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2017-Octob},
	year = {2017},
	pages = {2593{\textendash}2602},
	abstract = {Existing methods for 3D scene flow estimation often fail in the presence of large displacement or local ambiguities, e.g., at texture-less or reflective surfaces. However, these challenges are omnipresent in dynamic road scenes, which is the focus of this work. Our main contribution is to overcome these 3D motion estimation problems by exploiting recognition. In particular, we investigate the importance of recognition granularity, from coarse 2D bounding box estimates over 2D instance segmentations to fine-grained 3D object part predictions. We compute these cues using CNNs trained on a newly annotated dataset of stereo images and integrate them into a CRF-based model for robust 3D scene flow estimation - an approach we term Instance Scene Flow. We analyze the importance of each recognition cue in an ablation study and observe that the instance segmentation cue is by far strongest, in our setting. We demonstrate the effectiveness of our method on the challenging KITTI 2015 scene flow benchmark where we achieve state-of-the-art performance at the time of submission.},
	isbn = {9781538610329},
	issn = {15505499},
	doi = {10.1109/ICCV.2017.281},
	author = {Behl, Aseem and Omid Hosseini Jafari and Mustikovela, Siva Karthik and Hassan Abu Alhaija and Carsten Rother and Geiger, Andreas}
}
@conference {Schlesinger2017,
	title = {Crowd sourcing image segmentation with iaSTAPLE},
	booktitle = {Proceedings - International Symposium on Biomedical Imaging},
	year = {2017},
	pages = {401{\textendash}405},
	abstract = {We propose a novel label fusion technique as well as a crowdsourcing protocol to efficiently obtain accurate epithelial cell segmentations from non-expert crowd workers. Our label fusion technique simultaneously estimates the true segmentation, the performance levels of individual crowd workers, and an image segmentation model in the form of a pairwise Markov random field. We term our approach image-aware STAPLE (iaSTAPLE) since our image segmentation model seamlessly integrates into the well-known and widely used STAPLE approach. In an evaluation on a light microscopy dataset containing more than 5000 membrane labeled epithelial cells of a fly wing, we show that iaSTAPLE outperforms STAPLE in terms of segmentation accuracy as well as in terms of the accuracy of estimated crowd worker performance levels, and is able to correctly segment 99\% of all cells when compared to expert segmentations. These results show that iaSTAPLE is a highly useful tool for crowd sourcing image segmentation.},
	keywords = {Crowdsourcing, Epithelial cell segmentation, IaSTAPLE, Markovian Random Fields},
	isbn = {9781509011711},
	issn = {19458452},
	doi = {10.1109/ISBI.2017.7950547},
	author = {Schlesinger, Dmitrij and Jug, Florian and Myers, Gene and Carsten Rother and Kainmueller, Dagmar}
}
@conference {Ramos2017,
	title = {Detecting unexpected obstacles for self-driving cars: Fusing deep learning and geometric modeling},
	booktitle = {IEEE Intelligent Vehicles Symposium, Proceedings},
	year = {2017},
	month = {dec},
	pages = {1025{\textendash}1032},
	abstract = {The detection of small road hazards, such as lost cargo, is a vital capability for self-driving cars. We tackle this challenging and rarely addressed problem with a vision system that leverages appearance, contextual as well as geometric cues. To utilize the appearance and contextual cues, we propose a new deep learning-based obstacle detection framework. Here a variant of a fully convolutional network is proposed to predict a pixel-wise semantic labeling of (i) free-space, (ii) on-road unexpected obstacles, and (iii) background. The geometric cues are exploited using a state-of-The-Art detection approach that predicts obstacles from stereo input images via model-based statistical hypothesis tests. We present a principled Bayesian framework to fuse the semantic and stereo-based detection results. The mid-level Stixel representation is used to describe obstacles in a flexible, compact and robust manner. We evaluate our new obstacle detection system on the Lost and Found dataset, which includes very challenging scenes with obstacles of only 5 cm height. Overall, we report a major improvement over the state-of-The-Art, with a performance gain of 27.4\%. In particular, we achieve a detection rate of over 90\% for distances of up to 50 m. Our system operates at 22 Hz on our self-driving platform.},
	isbn = {9781509048045},
	doi = {10.1109/IVS.2017.7995849},
	url = {http://arxiv.org/abs/1612.06573},
	author = {Ramos, Sebastian and Gehrig, Stefan and Pinggera, Peter and Franke, Uwe and Carsten Rother}
}
@conference {Brachmann2017,
	title = {DSAC - Differentiable RANSAC for camera localization},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {nov},
	pages = {2492{\textendash}2500},
	abstract = {RANSAC is an important algorithm in robust optimization and a central building block for many computer vision applications. In recent years, traditionally hand-crafted pipelines have been replaced by deep learning pipelines, which can be trained in an end-to-end fashion. However, RANSAC has so far not been used as part of such deep learning pipelines, because its hypothesis selection procedure is non-differentiable. In this work, we present two different ways to overcome this limitation. The most promising approach is inspired by reinforcement learning, namely to replace the deterministic hypothesis selection by a probabilistic selection for which we can derive the expected loss w.r.t. to all learnable parameters. We call this approach DSAC, the differentiable counterpart of RANSAC. We apply DSAC to the problem of camera localization, where deep learning has so far failed to improve on traditional approaches. We demonstrate that by directly minimizing the expected loss of the output camera poses, robustly estimated by RANSAC, we achieve an increase in accuracy. In the future, any deep learning pipeline can use DSAC as a robust optimization component.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.267},
	url = {http://arxiv.org/abs/1611.05705},
	author = {Brachmann, Eric and Krull, Alexander and Nowozin, Sebastian and Shotton, Jamie and Michel, Frank and Gumhold, Stefan and Carsten Rother}
}
@conference {Michel2017,
	title = {Global hypothesis generation for 6D object pose estimation},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {dec},
	pages = {115{\textendash}124},
	abstract = {This paper addresses the task of estimating the 6D pose of a known 3D object from a single RGB-D image. Most modern approaches solve this task in three steps: i) Compute local features; ii) Generate a pool of pose-hypotheses; iii) Select and refine a pose from the pool. This work focuses on the second step. While all existing approaches generate the hypotheses pool via local reasoning, e.g. RANSAC or Hough-voting, we are the first to show that global reasoning is beneficial at this stage. In particular, we formulate a novel fully-connected Conditional Random Field (CRF) that outputs a very small number of pose-hypotheses. Despite the potential functions of the CRF being non-Gaussian, we give a new and efficient two-step optimization procedure, with some guarantees for optimality. We utilize our global hypotheses generation procedure to produce results that exceed state-of-the-art for the challenging "Occluded Object Dataset".},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.20},
	url = {http://arxiv.org/abs/1612.02287},
	author = {Michel, Frank and Kirillov, Alexander and Brachmann, Eric and Krull, Alexander and Gumhold, Stefan and Savchynskyy, Bogdan and Carsten Rother}
}
@conference {Kirillov2017a,
	title = {InstanceCut: From edges to instances with MultiCut},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	pages = {7322{\textendash}7331},
	abstract = {This work addresses the task of instance-aware semantic segmentation. Our key motivation is to design a simple method with a new modelling-paradigm, which therefore has a different trade-off between advantages and disadvantages compared to known approaches.Our approach, we term InstanceCut, represents the problem by two output modalities: (i) an instance-agnostic semantic segmentation and (ii) all instance-boundaries. The former is computed from a standard convolutional neural network for semantic segmentation, and the latter is derived from a new instanceaware edge detection model. To reason globally about the optimal partitioning of an image into instances, we combine these two modalities into a novel MultiCut formulation. We evaluate our approach on the challenging CityScapes dataset. Despite the conceptual simplicity of our approach, we achieve the best result among all published methods, and perform particularly well for rare object classes.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.774},
	author = {Kirillov, Alexander and Levinkov, Evgeny and Bj{\"o}rn Andres and Savchynskyy, Bogdan and Carsten Rother}
}
@conference {Levinkov2017,
	title = {Joint graph decomposition \& node labeling: Problem, algorithms, applications},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	pages = {1904{\textendash}1912},
	abstract = {We state a combinatorial optimization problem whose feasible solutions define both a decomposition and a node labeling of a given graph. This problem offers a common mathematical abstraction of seemingly unrelated computer vision tasks, including instance-separating semantic segmentation, articulated human body pose estimation and multiple object tracking. Conceptually, the problem we state generalizes the unconstrained integer quadratic program and the minimum cost lifted multicut problem, both of which are NP-hard. In order to find feasible solutions efficiently, we define two local search algorithms that converge monotonously to a local optimum, offering a feasible solution at any time. To demonstrate their effectiveness in tackling computer vision tasks, we apply these algorithms to instances of the problem that we construct from published data, using published algorithms. We report state-of-the-art application-specific accuracy for the three above-mentioned applications.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.206},
	author = {Levinkov, Evgeny and Uhrig, Jonas and Tang, Siyu and Omran, Mohamed and Insafutdinov, Eldar and Kirillov, Alexander and Carsten Rother and Brox, Thomas and Schiele, Bernt and Bj{\"o}rn Andres}
}
@conference {Kirillov2017,
	title = {Joint training of generic CNN-CRF models with stochastic optimization},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {10112 LNCS},
	year = {2017},
	pages = {221{\textendash}236},
	abstract = {We propose a new CNN-CRF end-to-end learning framework, which is based on joint stochastic optimization with respect to both Convolutional Neural Network (CNN) and Conditional Random Field (CRF) parameters. While stochastic gradient descent is a standard technique for CNN training, it was not used for joint models so far. We show that our learning method is (i) general, i.e. it applies to arbitrary CNN and CRF architectures and potential functions; (ii) scalable, i.e. it has a low memory footprint and straightforwardly parallelizes on GPUs; (iii) easy in implementation. Additionally, the unified CNN-CRF optimization approach simplifies a potential hardware implementation. We empirically evaluate our method on the task of semantic labeling of body parts in depth images and show that it compares favorably to competing techniques.},
	isbn = {9783319541839},
	issn = {16113349},
	doi = {10.1007/978-3-319-54184-6_14},
	url = {http://host.robots.ox.ac.uk:8080/leaderboard},
	author = {Kirillov, A and Schlesinger, D and Zheng, S and Savchynskyy, B and Torr, P. H.S. and Carsten Rother}
}
@conference {Kruse2017,
	title = {Learning to Push the Limits of Efficient FFT-Based Image Deconvolution},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2017-Octob},
	year = {2017},
	pages = {4596{\textendash}4604},
	abstract = {This work addresses the task of non-blind image deconvolution. Motivated to keep up with the constant increase in image size, with megapixel images becoming the norm, we aim at pushing the limits of efficient FFT-based techniques. Based on an analysis of traditional and more recent learning-based methods, we generalize existing discriminative approaches by using more powerful regularization, based on convolutional neural networks. Additionally, we propose a simple, yet effective, boundary adjustment method that alleviates the problematic circular convolution assumption, which is necessary for FFT-based deconvolution. We evaluate our approach on two common non-blind deconvolution benchmarks and achieve state-of-the-art results even when including methods which are computationally considerably more expensive.},
	isbn = {9781538610329},
	issn = {15505499},
	doi = {10.1109/ICCV.2017.491},
	author = {Kruse, Jakob and Carsten Rother and Schmidt, Uwe}
}
@techreport {Kruse,
	title = {Learning to Push the Limits of Efficient FFT-based Image Deconvolution - Supplemental Material},
	year = {2017},
	abstract = {1. Details about boundary adjustment comparison Section 4.3 of the main paper compares our proposed boundary adjustment (BA) strategy (Our BA, cf. Eq. 17 and Fig. 2 of the main paper) to the traditional edgetapering method (ET once, cf. Eq. 11 of the main paper) and the BA approach (ET each) of CSF [3]; these BA strategies are compared within our FDN model, the CSF model, and a standard Wiener filter [5]. Specifically, we use the publicly available code to train different variants of the CSF model on a dataset of the same size as ours, and only adjust the BA strategy. Furthermore, we apply the Wiener filter as defined in Eq. 2 of the main paper, which we can use iteratively with our BA approach by replacing y with ϕ t (y, k, x t); we estimate the expected image spectrum n from 3000 clean image patches. While our BA comparison is depicted visually in Fig. 5 of the main paper, Table 1 also provides the numeric results and additionally includes stages 6-10 of our FDN model. As compared to the BA approach of CSF (ET each), the results suggest that CSF would also benefit from further stages if used with our BA strategy (cf. 6 th column). Remarkably, the performance of the Wiener filter is not even fully saturated after 50 iterations (Wiener 50) when applied with our BA approach (cf. 3 rd column, only every 5 th step shown after iteration 10). Fig. 1 shows an example where our proposed BA strategy yields a substantial improvement in image quality compared to standard edgetapering (ET once).},
	author = {Kruse, Jakob and Carsten Rother and Schmidt, Uwe and Dresden, T U}
}
@conference {Krull2017,
	title = {PoseAgent: Budget-constrained 6D object pose estimation via reinforcement learning},
	booktitle = {Proceedings - 30th IEEE Conference on Computer Vision and Pattern Recognition, CVPR 2017},
	volume = {2017-Janua},
	year = {2017},
	month = {dec},
	pages = {2566{\textendash}2574},
	abstract = {State-of-the-art computer vision algorithms often achieve efficiency by making discrete choices about which hypotheses to explore next. This allows allocation of computational resources to promising candidates, however, such decisions are non-differentiable. As a result, these algorithms are hard to train in an end-to-end fashion. In this work we propose to learn an efficient algorithm for the task of 6D object pose estimation. Our system optimizes the parameters of an existing state-of-the art pose estimation system using reinforcement learning, where the pose estimation system now becomes the stochastic policy, parametrized by a CNN. Additionally, we present an efficient training algorithm that dramatically reduces computation time. We show empirically that our learned pose estimation procedure makes better use of limited resources and improves upon the state-of-the-art on a challenging dataset. Our approach enables differentiable end-to-end training of complex algorithmic pipelines and learns to make optimal use of a given computational budget.},
	isbn = {9781538604571},
	doi = {10.1109/CVPR.2017.275},
	url = {http://arxiv.org/abs/1612.03779},
	author = {Krull, Alexander and Brachmann, Eric and Nowozin, Sebastian and Michel, Frank and Shotton, Jamie and Carsten Rother}
}
@booklet {Massiceti,
	title = {Random Forests versus Neural Networks - What{\textquoteright}s best for camera location},
	year = {2017},
	abstract = {This work addresses the task of camera localiza-tion in a known 3D scene given a single input RGB image. State-of-the-art approaches accomplish this in two steps: firstly, regressing for every pixel in the image its 3D scene coordinate and subsequently, using these coordinates to estimate the final 6D camera pose via RANSAC. To solve the first step, Random Forests (RFs) are typically used. On the other hand, Neural Networks (NNs) reign in many dense regression tasks, but are not test-time efficient. We ask the question: which of the two is best for camera localization? To address this, we make two method contributions: (1) a test-time efficient NN architecture which we term a ForestNet that is derived and initialized from a RF, and (2) a new fully-differentiable robust averaging technique for regression ensembles which can be trained end-to-end with a NN. Our experimental findings show that for scene coordinate regression, traditional NN architectures are superior to test-time efficient RFs and ForestNets, however, this does not translate to final 6D camera pose accuracy where RFs and ForestNets perform slightly better. To summarize, our best method, a ForestNet with a robust average, which has an equivalent fast and lightweight RF, improves over the state-of-the-art for camera localization on the 7-Scenes dataset [1]. While this work focuses on scene coordinate regression for camera localization, our innovations may also be applied to other continuous regression tasks.},
	author = {Massiceti, Daniela and Krull, Alexander and Brachmann, Eric and Carsten Rother and Torr, Philip H S}
}
@article {Hullin2017,
	title = {Semantic-Aware Image Smoothing},
	journal = {Vision, Modeling, and Visualization},
	year = {2017},
	abstract = {Structure-preserving image smoothing aims to extract semantically meaningful image structure from texture, which is one of the fundamental problems in computer vision and graphics. However, it is still not clear how to define this concept. On the other hand, semantic image labeling has achieved significant progress recently and has been widely used in many computer vision tasks. In this paper, we present an interesting observation, i.e. high-level semantic image labeling information can provide a meaningful structure prior naturally. Based on this observation, we propose a simple and yet effective method, which we term semantic smoothing, by exploiting the semantic information to accomplish semantically structure-preserving image smoothing. We show that our approach outperforms the state-of-the-art approaches in texture removal by considering the semantic infor-mation for structure preservation. Also, we apply our approach to three applications: detail enhancement, edge detection, and image segmentation, and we demonstrate the effectiveness of our semantic smoothing method on these problems.},
	keywords = {Enhancement{\textemdash} Smoothing, I43 [Image Processing and Computer Vision]},
	url = {https://hci.iwr.uni-heidelberg.de/vislearn/wp-content/uploads/2014/08/paper1024_CRC.pdf},
	author = {Hullin, M and Klein, R and Schultz, T and Yao, A and Weihao Li and Omid Hosseini Jafari and Carsten Rother}
}
@conference {Mustikovela2016,
	title = {Can ground truth label propagation from video help semantic segmentation?},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9915 LNCS},
	year = {2016},
	pages = {804{\textendash}820},
	abstract = {For state-of-the-art semantic segmentation task, training convolutional neural networks (CNNs) requires dense pixelwise ground truth (GT) labeling, which is expensive and involves extensive human effort. In this work, we study the possibility of using auxiliary ground truth, so-called pseudo ground truth (PGT) to improve the performance. The PGT is obtained by propagating the labels of a GT frame to its subsequent frames in the video using a simple CRF-based, cue integration framework. Our main contribution is to demonstrate the use of noisy PGT along with GT to improve the performance of a CNN. We perform a systematic analysis to find the right kind of PGT that needs to be added along with the GT for training a CNN. In this regard, we explore three aspects of PGT which influence the learning of a CNN: (i) the PGT labeling has to be of good quality; (ii) the PGT images have to be different compared to the GT images; (iii) the PGT has to be trusted differently than GT. We conclude that PGT which is diverse from GT images and has good quality of labeling can indeed help improve the performance of a CNN. Also, when PGT is multiple folds larger than GT, weighing down the trust on PGT helps in improving the accuracy. Finally, We show that using PGT along with GT, the performance of Fully Convolutional Network (FCN) on Camvid data is increased by 2.7\% on IoU accuracy. We believe such an approach can be used to train CNNs for semantic video segmentation where sequentially labeled image frames are needed. To this end, we provide recommendations for using PGT strategically for semantic segmentation and hence bypass the need for extensive human efforts in labeling.},
	isbn = {9783319494081},
	issn = {16113349},
	doi = {10.1007/978-3-319-49409-8_66},
	author = {Mustikovela, Siva Karthik and Yang, Michael Ying and Carsten Rother}
}
@conference {Royer2016,
	title = {Convexity shape constraints for image segmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {2016-Decem},
	year = {2016},
	month = {sep},
	pages = {402{\textendash}410},
	abstract = {Segmenting an image into multiple components is a central task in computer vision. In many practical scenarios, prior knowledge about plausible components is available. Incorporating such prior knowledge into models and algorithms for image segmentation is highly desirable, yet can be non-trivial. In this work, we introduce a new approach that allows, for the first time, to constrain some or all components of a segmentation to have convex shapes. Specifically, we extend the Minimum Cost Multicut Problem by a class of constraints that enforce convexity. To solve instances of this NP-hard integer linear program to optimality, we separate the proposed constraints in the branch-and-cut loop of a state-of-the-art ILP solver. Results on photographs and micrographs demonstrate the effectiveness of the approach as well as its advantages over the state-of-the-art heuristic.},
	isbn = {9781467388504},
	issn = {10636919},
	doi = {10.1109/CVPR.2016.50},
	url = {http://arxiv.org/abs/1509.02122},
	author = {Royer, Loic A. and Richmond, David L. and Carsten Rother and Bj{\"o}rn Andres and Kainmueller, Dagmar}
}
@conference {Mund2016,
	title = {Introducing LiDAR Point Cloud-based Object Classification for Safer Apron Operations},
	booktitle = {International Symposium on Enhanced Solutions for Aircraft and Vehicle Surveillance Applications},
	year = {2016},
	abstract = {Current procedures for conventional and remote airport ground control still rely on the direct (camera-) view. Despite further support by different Radar applications occasional shortcomings in the awareness of the responsible controllers may occur, particularly under adverse weather conditions, giving rise to capacity backlogs, incidents and accidents. As Laser scanners and computer vision algorithms have reached new performance levels in recent years, we proposed a novel concept for complete and independent airport apron surveillance based on LiDAR 3D point data. In this paper we extend our object detection/segmentation technique by addressing object classification in LiDAR 3D scans. We hereby enable LiDAR{\textquoteleft}s unique capability to classify non-cooperative objects by means of a single sensor and learned model knowledge. Our technique was able to classify and to estimate the poses of an Airbus A319-100 and a Boeing B737-700 parked on the airport apron. In the future we will enhance our classification technique to a wider range of objects including moving ground vehicles and pedestrians.},
	keywords = {- lidar, 3d point cloud, aircraft, airport ground surveillance, apron control, apron control rely on, classification, current legacy procedures for, laser scanning, pose estimation},
	url = {https://goo.gl/28Yoqh},
	author = {Mund, Johannes and Michel, Frank and Dieke-Meier, Franziska and Fricke, Hartmut and Meyer, Lothar and Carsten Rother}
}
@conference {Pinggera2016,
	title = {Lost and found: Detecting small road hazards for self-driving vehicles},
	booktitle = {IEEE International Conference on Intelligent Robots and Systems},
	volume = {2016-Novem},
	year = {2016},
	pages = {1099{\textendash}1106},
	abstract = {Detecting small obstacles on the road ahead is a critical part of the driving task which has to be mastered by fully autonomous cars. In this paper, we present a method based on stereo vision to reliably detect such obstacles from a moving vehicle. The proposed algorithm performs statistical hypothesis tests in disparity space directly on stereo image data, assessing freespace and obstacle hypotheses on independent local patches. This detection approach does not depend on a global road model and handles both static and moving obstacles. For evaluation, we employ a novel lost-cargo image sequence dataset comprising more than two thousand frames with pixelwise annotations of obstacle and free-space and provide a thorough comparison to several stereo-based baseline methods. The dataset will be made available to the community to foster further research on this important topic4. The proposed approach outperforms all considered baselines in our evaluations on both pixel and object level and runs at frame rates of up to 20 Hz on 2 mega-pixel stereo imagery. Small obstacles down to the height of 5 cm can successfully be detected at 20 m distance at low false positive rates.},
	isbn = {9781509037629},
	issn = {21530866},
	doi = {10.1109/IROS.2016.7759186},
	url = {http://www.6d-vision.com/lostandfounddataset},
	author = {Pinggera, Peter and Ramos, Sebastian and Gehrig, Stefan and Franke, Uwe and Carsten Rother and Mester, Rudolf}
}
@conference {Richmond2016b,
	title = {Mapping auto-context decision forests to deep convnets for semantic segmentation},
	booktitle = {British Machine Vision Conference 2016, BMVC 2016},
	volume = {2016-Septe},
	year = {2016},
	pages = {144.1{\textendash}144.12},
	abstract = {We consider the task of pixel-wise semantic segmentation given a small set of labeled training images. Among two of the most popular techniques to address this task are Random Forests (RF) and Neural Networks (NN). In this work, we explore the relationship between two special forms of these techniques: stacked RFs (namely Auto-context) and deep Convolutional Neural Networks (ConvNet). Our main contribution is to show that Auto-context can be mapped to a deep ConvNet with novel architecture, and thereby trained end-to-end. This mapping can be viewed as an intelligent initialization of a deep ConvNet, enabling training even in the face of very limited amounts of training data. We also demonstrate an approximate mapping back from the refined ConvNet to a second stacked RF, with improved performance over the original. We experimentally verify that these mappings outperform stacked RFs for two different applications in computer vision and biology: Kinect-based body part labeling from depth images, and somite segmentation in microscopy images of developing zebrafish.},
	doi = {10.5244/C.30.144},
	author = {Richmond, David L and Kainmueller, Dagmar and Yang, Michael Y and Myers, Eugene W and Carsten Rother}
}
@conference {Richmond2016a,
	title = {Mapping auto-context decision forests to deep convnets for semantic segmentation},
	booktitle = {British Machine Vision Conference 2016, BMVC 2016},
	volume = {2016-Septe},
	year = {2016},
	pages = {144.1{\textendash}144.12},
	abstract = {We consider the task of pixel-wise semantic segmentation given a small set of labeled training images. Among two of the most popular techniques to address this task are Random Forests (RF) and Neural Networks (NN). In this work, we explore the relationship between two special forms of these techniques: stacked RFs (namely Auto-context) and deep Convolutional Neural Networks (ConvNet). Our main contribution is to show that Auto-context can be mapped to a deep ConvNet with novel architecture, and thereby trained end-to-end. This mapping can be viewed as an intelligent initialization of a deep ConvNet, enabling training even in the face of very limited amounts of training data. We also demonstrate an approximate mapping back from the refined ConvNet to a second stacked RF, with improved performance over the original. We experimentally verify that these mappings outperform stacked RFs for two different applications in computer vision and biology: Kinect-based body part labeling from depth images, and somite segmentation in microscopy images of developing zebrafish.},
	doi = {10.5244/C.30.144},
	url = {https://github.com/BVLC/caffe/wiki/Model-Zoo\#fcn},
	author = {Richmond, David L and Kainmueller, Dagmar and Yang, Michael Y and Myers, Eugene W and Carsten Rother}
}
@conference {Richmond2016,
	title = {Mapping auto-context decision forests to deep convnets for semantic segmentation},
	booktitle = {British Machine Vision Conference 2016, BMVC 2016},
	volume = {2016-Septe},
	year = {2016},
	month = {jul},
	pages = {144.1{\textendash}144.12},
	abstract = {We consider the task of pixel-wise semantic segmentation given a small set of labeled training images. Among two of the most popular techniques to address this task are Random Forests (RF) and Neural Networks (NN). In this work, we explore the relationship between two special forms of these techniques: stacked RFs (namely Auto-context) and deep Convolutional Neural Networks (ConvNet). Our main contribution is to show that Auto-context can be mapped to a deep ConvNet with novel architecture, and thereby trained end-to-end. This mapping can be viewed as an intelligent initialization of a deep ConvNet, enabling training even in the face of very limited amounts of training data. We also demonstrate an approximate mapping back from the refined ConvNet to a second stacked RF, with improved performance over the original. We experimentally verify that these mappings outperform stacked RFs for two different applications in computer vision and biology: Kinect-based body part labeling from depth images, and somite segmentation in microscopy images of developing zebrafish.},
	doi = {10.5244/C.30.144},
	url = {http://arxiv.org/abs/1507.07583},
	author = {Richmond, David L. and Kainmueller, Dagmar and Yang, Michael Y. and Myers, Eugene W. and Carsten Rother}
}
@conference {Sellent2016,
	title = {Stereo video deblurring},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9906 LNCS},
	year = {2016},
	pages = {558{\textendash}575},
	abstract = {Videos acquired in low-light conditions often exhibit motion blur, which depends on the motion of the objects relative to the camera. This is not only visually unpleasing, but can hamper further processing. With this paper we are the first to show how the availability of stereo video can aid the challenging video deblurring task.We leverage 3D scene flow, which can be estimated robustly even under adverse conditions. We go beyond simply determining the object motion in two ways: First, we show how a piecewise rigid 3D scene flow representation allows to induce accurate blur kernels via local homographies. Second, we exploit the estimated motion boundaries of the 3D scene flow to mitigate ringing artifacts using an iterative weighting scheme. Being aware of 3D object motion, our approach can deal robustly with an arbitrary number of independently moving objects. We demonstrate its benefit over state-ofthe- art video deblurring using quantitative and qualitative experiments on rendered scenes and real videos.},
	keywords = {Object motion blur, Scene flow, Spatially-variant deblurring},
	isbn = {9783319464749},
	issn = {16113349},
	doi = {10.1007/978-3-319-46475-6_35},
	author = {Sellent, Anita and Carsten Rother and Roth, Stefan}
}
@techreport {Sellent,
	title = {Stereo Video Deblurring-Supplemental Material},
	year = {2016},
	abstract = {One of the key contributions in our stereo video deblurring is to employ 3D scene flow to induce blur kernels based on homographies. As the difference to inducing blur kernels from an optical flow field may seem subtle but increases deblurring performance considerably, we schematically illustrate the difference of the two ways of generating blur kernels in Fig. 10. (a) 3D motion (b) Image plane projection: Blur kernels from linear displacements (c) Image plane projection: Blur kernels from homographies Fig. 10. Inducing blur matrices: (a) Assume that 3D point P moves with a constant rigid body motion in 3D, e. g. with a yaw motion. The projection of this motion to the image plane (blue) is a circular trajectory. The corresponding 2D ground truth displacement (yellow), however, is a vector in the image plane that connects start and end point of the motion during a time interval. (b) Using optical flow, the blur kernel at a location z is approximated by identifying all pixels that, according to their spatially-variant displacement, pass through z during the time interval. The image content at point x is correctly identified as passing through z. The image content at point y is not identified correctly as its 2D displacement passes z at a distance. Instead, the image content at point{\textasciicircum}ypoint{\textasciicircum} point{\textasciicircum}y is erroneously identified as passing through z, even though{\textasciicircum}ythough{\textasciicircum} though{\textasciicircum}y has a different distance to the rotation center than z. This results in the distorted kernels shown in Fig. 3b. (c) Assuming 3D points in the vicinity of P to form a plane, we employ 3D homographies to generate blur kernels. The blur kernel at location z is thus formed by the image points whose trajectory, according to the homography, passes through z during the time interval. Consequently, y is correctly identified as passing through z, leading to the kernels shaped as circular arcs shown in Fig. 3a},
	author = {Sellent, Anita and Carsten Rother and Roth, Stefan}
}
@conference {Brachmann2016a,
	title = {Uncertainty-Driven 6D Pose Estimation of Objects and Scenes from a Single RGB Image},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {2016-Decem},
	year = {2016},
	pages = {3364{\textendash}3372},
	abstract = {In recent years, the task of estimating the 6D pose of object instances and complete scenes, i.e. camera localization, from a single input image has received considerable attention. Consumer RGB-D cameras have made this feasible, even for difficult, texture-less objects and scenes. In this work, we show that a single RGB image is sufficient to achieve visually convincing results. Our key concept is to model and exploit the uncertainty of the system at all stages of the processing pipeline. The uncertainty comes in the form of continuous distributions over 3D object coordinates and discrete distributions over object labels. We give three technical contributions. Firstly, we develop a regularized, auto-context regression framework which iteratively reduces uncertainty in object coordinate and object label predictions. Secondly, we introduce an efficient way to marginalize object coordinate distributions over depth. This is necessary to deal with missing depth information. Thirdly, we utilize the distributions over object labels to detect multiple objects simultaneously with a fixed budget of RANSAC hypotheses. We tested our system for object pose estimation and camera localization on commonly used data sets. We see a major improvement over competing systems.},
	isbn = {9781467388504},
	issn = {10636919},
	doi = {10.1109/CVPR.2016.366},
	author = {Brachmann, Eric and Michel, Frank and Krull, Alexander and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Brachmann2016,
	title = {Uncertainty-Driven 6D Pose Estimation of Objects and Scenes from a Single RGB Image},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {2016-Decem},
	year = {2016},
	pages = {3364{\textendash}3372},
	abstract = {In recent years, the task of estimating the 6D pose of object instances and complete scenes, i.e. camera localization, from a single input image has received considerable attention. Consumer RGB-D cameras have made this feasible, even for difficult, texture-less objects and scenes. In this work, we show that a single RGB image is sufficient to achieve visually convincing results. Our key concept is to model and exploit the uncertainty of the system at all stages of the processing pipeline. The uncertainty comes in the form of continuous distributions over 3D object coordinates and discrete distributions over object labels. We give three technical contributions. Firstly, we develop a regularized, auto-context regression framework which iteratively reduces uncertainty in object coordinate and object label predictions. Secondly, we introduce an efficient way to marginalize object coordinate distributions over depth. This is necessary to deal with missing depth information. Thirdly, we utilize the distributions over object labels to detect multiple objects simultaneously with a fixed budget of RANSAC hypotheses. We tested our system for object pose estimation and camera localization on commonly used data sets. We see a major improvement over competing systems.},
	isbn = {9781467388504},
	issn = {10636919},
	doi = {10.1109/CVPR.2016.366},
	author = {Brachmann, Eric and Michel, Frank and Krull, Alexander and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@article {kappes_15_comparative,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {International Journal of Computer Vision},
	year = {2015},
	note = {1},
	pages = {1-30},
	doi = {10.1007/s11263-015-0809-x},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, S. and Dhruv Batra and Kim, S. and Bernhard X. Kausler and Thorben Kr{\"o}ger and Lellmann, J. and Komodakis, N. and Savchynskyy, B. and Carsten Rother}
}
@article {Kappes2015b,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {International Journal of Computer Vision},
	volume = {115},
	number = {2},
	year = {2015},
	pages = {155{\textendash}184},
	abstract = {Szeliski et al. published an influential study in 2006 on energy minimization methods for Markov random fields. This study provided valuable insights in choosing the best optimization technique for certain classes of problems. While these insights remain generally useful today, the phenomenal success of random field models means that the kinds of inference problems that have to be solved changed significantly. Specifically, the models today often include higher order interactions, flexible connectivity structures, large label-spaces of different cardinalities, or learned energy tables. To reflect these changes, we provide a modernized and enlarged study. We present an empirical comparison of more than 27 state-of-the-art optimization techniques on a corpus of 2453 energy minimization instances from diverse applications in computer vision. To ensure reproducibility, we evaluate all methods in the OpenGM 2 framework and report extensive results regarding runtime and solution quality. Key insights from our study agree with the results of Szeliski et al. for the types of models they studied. However, on new and challenging types of models our findings disagree and suggest that polyhedral methods and integer programming solvers are competitive in terms of runtime and solution quality over a large range of model types.},
	keywords = {Benchmark, Combinatorial optimization, Discrete graphical models},
	issn = {15731405},
	doi = {10.1007/s11263-015-0809-x},
	author = {Kappes, J{\"o}rg H and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, Sebastian and Dhruv Batra and Kim, Sungwoong and Kausler, Bernhard X and Kr{\"o}ger, Thorben and Lellmann, Jan and Komodakis, Nikos and Savchynskyy, Bogdan and Carsten Rother}
}
@article {Kappes2015a,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {International Journal of Computer Vision},
	volume = {115},
	number = {2},
	year = {2015},
	pages = {155{\textendash}184},
	abstract = {Szeliski et al. published an influential study in 2006 on energy minimization methods for Markov random fields. This study provided valuable insights in choosing the best optimization technique for certain classes of problems. While these insights remain generally useful today, the phenomenal success of random field models means that the kinds of inference problems that have to be solved changed significantly. Specifically, the models today often include higher order interactions, flexible connectivity structures, large label-spaces of different cardinalities, or learned energy tables. To reflect these changes, we provide a modernized and enlarged study. We present an empirical comparison of more than 27 state-of-the-art optimization techniques on a corpus of 2453 energy minimization instances from diverse applications in computer vision. To ensure reproducibility, we evaluate all methods in the OpenGM 2 framework and report extensive results regarding runtime and solution quality. Key insights from our study agree with the results of Szeliski et al. for the types of models they studied. However, on new and challenging types of models our findings disagree and suggest that polyhedral methods and integer programming solvers are competitive in terms of runtime and solution quality over a large range of model types.},
	keywords = {Benchmark, Combinatorial optimization, Discrete graphical models},
	isbn = {25164671.25},
	issn = {15731405},
	doi = {10.1007/s11263-015-0809-x},
	author = {Kappes, J{\"o}rg H and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, Sebastian and Dhruv Batra and Kim, Sungwoong and Kausler, Bernhard X and Kr{\"o}ger, Thorben and Lellmann, Jan and Komodakis, Nikos and Savchynskyy, Bogdan and Carsten Rother}
}
@article {Kappes2015,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {International Journal of Computer Vision},
	volume = {115},
	number = {2},
	year = {2015},
	pages = {155{\textendash}184},
	abstract = {Szeliski et al. published an influential study in 2006 on energy minimization methods for Markov random fields. This study provided valuable insights in choosing the best optimization technique for certain classes of problems. While these insights remain generally useful today, the phenomenal success of random field models means that the kinds of inference problems that have to be solved changed significantly. Specifically, the models today often include higher order interactions, flexible connectivity structures, large label-spaces of different cardinalities, or learned energy tables. To reflect these changes, we provide a modernized and enlarged study. We present an empirical comparison of more than 27 state-of-the-art optimization techniques on a corpus of 2453 energy minimization instances from diverse applications in computer vision. To ensure reproducibility, we evaluate all methods in the OpenGM 2 framework and report extensive results regarding runtime and solution quality. Key insights from our study agree with the results of Szeliski et al. for the types of models they studied. However, on new and challenging types of models our findings disagree and suggest that polyhedral methods and integer programming solvers are competitive in terms of runtime and solution quality over a large range of model types.},
	keywords = {Benchmark, Combinatorial optimization, Discrete graphical models},
	issn = {15731405},
	doi = {10.1007/s11263-015-0809-x},
	url = {http://hci.iwr.uni-heidelberg.de/opengm2/},
	author = {Kappes, J{\"o}rg H and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, Sebastian and Dhruv Batra and Kim, Sungwoong and Kausler, Bernhard X and Kr{\"o}ger, Thorben and Lellmann, Jan and Komodakis, Nikos and Savchynskyy, Bogdan and Carsten Rother}
}
@article {Kappes2015,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {Int.~J.~Comp.~Vision},
	year = {2015},
	note = {in press (preprint: arXiv:1404.0533)},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, S. and Dhruv Batra and Kim, S. and Bernhard X. Kausler and Thorben Kr{\"o}ger and Lellmann, J. and Komodakis, N. and Savchynskyy, B. and Carsten Rother}
}
@conference {Alhaija2015,
	title = {Graphflow{\textemdash}6D large displacement scene flow via graph matching},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9358},
	year = {2015},
	pages = {285{\textendash}296},
	abstract = {We present an approach for computing dense scene flow from two large displacement RGB-D images. When dealing with large displacements the crucial step is to estimate the overall motion correctly. While state-of-the-art approaches focus on RGB information to establish guiding correspondences, we explore the power of depth edges. To achieve this, we present a new graph matching technique that brings sparse depth edges into correspondence. An additional contribution is the formulation of a continuous-label energy which is used to densify the sparse graph matching output. We present results on challenging Kinect images, for which we outperform state-of-the-art techniques.},
	isbn = {9783319249469},
	issn = {16113349},
	doi = {10.1007/978-3-319-24947-6_23},
	author = {Hassan Abu Alhaija and Sellent, Anita and Kondermann, Daniel and Carsten Rother}
}
@conference {Kirillov2015a,
	title = {Inferring M-best diverse labelings in a single one},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2015 Inter},
	year = {2015},
	pages = {1814{\textendash}1822},
	abstract = {We consider the task of finding M-best diverse solutions in a graphical model. In a previous work by Batra et al. an algorithmic approach for finding such solutions was proposed, and its usefulness was shown in numerous applications. Contrary to previous work we propose a novel formulation of the problem in form of a single energy minimization problem in a specially constructed graphical model. We show that the method of Batra et al. can be considered as a greedy approximate algorithm for our model, whereas we introduce an efficient specialized optimization technique for it, based on alpha-expansion. We evaluate our method on two application scenarios, interactive and semantic image segmentation, with binary and multiple labels. In both cases we achieve considerably better error rates than state-of-the art diversity methods. Furthermore, we empirically discover that in the binary label case we were able to reach global optimality for all test instances.},
	isbn = {9781467383912},
	issn = {15505499},
	doi = {10.1109/ICCV.2015.211},
	author = {Kirillov, Alexander and Savchynskyy, Bogdan and Schlesinger, Dmitrij and Vetrov, Dmitry and Carsten Rother}
}
@conference {Schelten2015,
	title = {Interleaved regression tree field cascades for blind image deconvolution},
	booktitle = {Proceedings - 2015 IEEE Winter Conference on Applications of Computer Vision, WACV 2015},
	year = {2015},
	pages = {494{\textendash}501},
	abstract = {Image blur from camera shake is a common cause for poor image quality in digital photography, prompting a significant recent interest in image deblurring. The vast majority of work on blind deblurring splits the problem into two subsequent steps: First, the blur process (i.e., blur kernel) is estimated, then the image is restored given the estimated kernel using a non-blind deblurring algorithm. Recent work in non-blind deblurring has shown that discriminative approaches can have clear image quality and runtime benefits over typical generative formulations. In this paper, we propose a cascade for blind deblurring that alternates between kernel estimation and discriminative deblurring using regression tree fields (RTFs). We further contribute a new dataset of realistic image blur kernels from human camera shake, which we use to train the discriminative component. Extensive qualitative and quantitative experiments show a clear gain in image quality by interleaving kernel estimation and discriminative deblurring in an iterative cascade.},
	isbn = {9781479966820},
	doi = {10.1109/WACV.2015.72},
	author = {Schelten, Kevin and Nowozin, Sebastian and Jancsary, Jeremy and Carsten Rother and Roth, Stefan}
}
@conference {Krull2015,
	title = {Learning analysis-by-synthesis for 6d pose estimation in RGB-D images},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2015 Inter},
	year = {2015},
	pages = {954{\textendash}962},
	abstract = {Analysis-by-synthesis has been a successful approach for many tasks in computer vision, such as 6D pose estimation of an object in an RGB-D image which is the topic of this work. The idea is to compare the observation with the output of a forward process, such as a rendered image of the object of interest in a particular pose. Due to occlusion or complicated sensor noise, it can be difficult to perform this comparison in a meaningful way. We propose an approach that "learns to compare", while taking these difficulties into account. This is done by describing the posterior density of a particular object pose with a convolutional neural network (CNN) that compares observed and rendered images. The network is trained with the maximum likelihood paradigm. We observe empirically that the CNN does not specialize to the geometry or appearance of specific objects. It can be used with objects of vastly different shapes and appearances, and in different backgrounds. Compared to state-of-the-art, we demonstrate a significant improvement on two different datasets which include a total of eleven objects, cluttered background, and heavy occlusion.},
	isbn = {9781467383912},
	issn = {15505499},
	doi = {10.1109/ICCV.2015.115},
	author = {Krull, Alexander and Brachmann, Eric and Michel, Frank and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Kirillov2015,
	title = {M-best-diverse labelings for submodular energies and beyond},
	booktitle = {Advances in Neural Information Processing Systems},
	volume = {2015-Janua},
	year = {2015},
	pages = {613{\textendash}621},
	abstract = {We consider the problem of findingM best diverse solutions of energy minimization problems for graphical models. Contrary to the sequential method of Batra et al., which greedily finds one solution after another, we infer all M solutions jointly. It was shown recently that such jointly inferred labelings not only have smaller total energy but also qualitatively outperform the sequentially obtained ones. The only obstacle for using this new technique is the complexity of the corresponding inference problem, since it is considerably slower algorithm than the method of Batra et al. In this work we show that the joint inference of M best diverse solutions can be formulated as a submodular energy minimization if the original MAP-inference problem is submodular, hence fast inference techniques can be used. In addition to the theoretical results we provide practical algorithms that outperform the current state-of-the-art and can be used in both submodular and non-submodular case.},
	issn = {10495258},
	author = {Kirillov, Alexander and Schlesinger, Dmitrij and Vetrov, Dmitry and Carsten Rother and Savchynskyy, Bogdan}
}
@conference {Zheng2015,
	title = {Object proposals estimation in depth image using compact 3D shape manifolds},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9358},
	year = {2015},
	pages = {196{\textendash}208},
	abstract = {Man-made objects, such as chairs, often have very large shape variations, making it challenging to detect them. In this work we investigate the task of finding particular object shapes from a single depth image. We tackle this task by exploiting the inherently low dimensionality in the object shape variations, which we discover and encode as a compact shape space. Starting from any collection of 3D models, we first train a low dimensional Gaussian Process Latent Variable Shape Space. We then sample this space, effectively producing infinite amounts of shape variations, which are used for training. Additionally, to support fast and accurate inference, we improve the standard 3D object category proposal generation pipeline by applying a shallow convolutional neural network-based filtering stage. This combination leads to considerable improvements for proposal generation, in both speed and accuracy. We compare our full system to previous state-of-the-art approaches, on four different shape classes, and show a clear improvement.},
	isbn = {9783319249469},
	issn = {16113349},
	doi = {10.1007/978-3-319-24947-6_16},
	author = {Zheng, Shuai and Prisacariu, Victor Adrian and Averkiou, Melinos and Cheng, Ming Ming and Mitra, Niloy J and Shotton, Jamie and Torr, Philip H.S. and Carsten Rother}
}
@article {Mitra2015,
	title = {Pacific Graphics 2015 DenseCut: Densely Connected CRFs for Realtime GrabCut},
	volume = {34},
	number = {7},
	year = {2015},
	abstract = {Figure-ground segmentation from bounding box input, provided either automatically or manually, has been ex-tremely popular in the last decade and influenced various applications. A lot of research has focused on high-quality segmentation, using complex formulations which often lead to slow techniques, and often hamper practi-cal usage. In this paper we demonstrate a very fast segmentation technique which still achieves very high quality results. We propose to replace the time consuming iterative refinement of global colour models in traditional GrabCut formulation by a densely connected CRF. To motivate this decision, we show that a dense CRF implicitly models unnormalized global colour models for foreground and background. Such relationship provides insightful analysis to bridge between dense CRF and GrabCut functional. We extensively evaluate our algorithm using two famous benchmarks. Our experimental results demonstrated that the proposed algorithm achieves an order of magnitude (10{\texttimes}) speed-up with respect to the closest competitor, and at the same time achieves a considerably higher accuracy.},
	keywords = {I46 [IMAGE PROCESSING AND COMPUTER VISION], partitioning, Segmentation{\textemdash}Region growing},
	url = {http://mftp.mmcheng.net/Papers/DenseCut.pdf},
	author = {Mitra, N J and Stam, J and Xu, K and Cheng, Ming-Ming and Prisacariu, Victor Adrian and Zheng, Shuai and Torr, Philip H S and Carsten Rother}
}
@conference {Mund2015a,
	title = {Performance evaluation of LiDAR point clouds towards automated FOD detection on airport aprons},
	booktitle = {Proceedings of ATACCS 2015 - 5th International Conference on Application and Theory of Automation in Command and Control Systems},
	year = {2015},
	pages = {85{\textendash}94},
	abstract = {Both the current system of airport ground control and the continuous implementation efforts of A-SMGCS and Remote Tower concepts require complete and independent surveillance coverage in real-time. We believe that 3D point clouds generated by an actively scanning LiDAR system available at TU Dresden may satisfy these high standards. Nonetheless, the utilization of LiDAR sensing for airport ground surveillance purposes is extremely challenging due to the unique requirement profile in this domain. This is also the reason why existing solutions in other domains such as autonomous driving and robotics are not directly applicable for airport ground surveillance. In a first step, we developed point cloud object detection and segmentation techniques to present that new data comprehensively to the airport apron controller. In this paper, we focused on the timely detection of dislocated objects (foreign object debris, forgotten equipment etc.) as a serious cause to hazardous situations on airport movement areas. The results are promising for various reference targets. However, the detection of very small objects (e.g. socket wrench) requires more elaborate algorithms to take full advantage of the current LiDAR technology. In the future we will assess the strength of LiDAR-based surveillance in terms of the number of hazardous situations that could be avoided or safely managed by the apron controller.},
	keywords = {Airport surveillance, apron control, Apron management service, FOD, Foreign object debris, laser scanning, LiDAR, Object detection, Point cloud},
	isbn = {9781450335621},
	doi = {10.1145/2899361.2899370},
	author = {Mund, Johannes and Zouhar, Alexander and Meyer, Lothar and Fricke, Hartmut and Carsten Rother}
}
@conference {Mund2015,
	title = {Performance evaluation of LiDAR point clouds towards automated FOD detection on airport aprons},
	booktitle = {Proceedings of ATACCS 2015 - 5th International Conference on Application and Theory of Automation in Command and Control Systems},
	year = {2015},
	pages = {85{\textendash}94},
	abstract = {Both the current system of airport ground control and the continuous implementation efforts of A-SMGCS and Remote Tower concepts require complete and independent surveillance coverage in real-time. We believe that 3D point clouds generated by an actively scanning LiDAR system available at TU Dresden may satisfy these high standards. Nonetheless, the utilization of LiDAR sensing for airport ground surveillance purposes is extremely challenging due to the unique requirement profile in this domain. This is also the reason why existing solutions in other domains such as autonomous driving and robotics are not directly applicable for airport ground surveillance. In a first step, we developed point cloud object detection and segmentation techniques to present that new data comprehensively to the airport apron controller. In this paper, we focused on the timely detection of dislocated objects (foreign object debris, forgotten equipment etc.) as a serious cause to hazardous situations on airport movement areas. The results are promising for various reference targets. However, the detection of very small objects (e.g. socket wrench) requires more elaborate algorithms to take full advantage of the current LiDAR technology. In the future we will assess the strength of LiDAR-based surveillance in terms of the number of hazardous situations that could be avoided or safely managed by the apron controller.},
	keywords = {Airport surveillance, apron control, Apron management service, FOD, Foreign object debris, laser scanning, LiDAR, Object detection, Point cloud},
	isbn = {9781450335621},
	doi = {10.1145/2899361.2899370},
	author = {Mund, Johannes and Zouhar, Alexander and Meyer, Lothar and Fricke, Hartmut and Carsten Rother}
}
@conference {Michel2015,
	title = {Pose Estimation of Kinematic Chain Instances via Object Coordinate Regression},
	year = {2015},
	pages = {181.1{\textendash}181.11},
	abstract = {In this paper, we address the problem of one shot pose estimation of articulated ob-jects from an RGB-D image. In particular, we consider object instances with the topol-ogy of a kinematic chain, i.e. assemblies of rigid parts connected by prismatic or revolute joints. This object type occurs often in daily live, for instance in the form of furniture or electronic devices. Instead of treating each object part separately we are using the rela-tionship between parts of the kinematic chain and propose a new minimal pose sampling approach. This enables us to create a pose hypothesis for a kinematic chain consist-ing of K parts by sampling K 3D-3D point correspondences. To asses the quality of our method, we gathered a large dataset containing four objects and 7000+ annotated RGB-D frames 1 . On this dataset we achieve considerably better results than a modified state-of-the-art pose estimation system for rigid objects.},
	doi = {10.5244/c.29.181},
	author = {Michel, Frank and Krull, Alexander and Brachmann, Eric and Yang, Michael Ying and Gumhold, Stefan and Carsten Rother}
}
@conference {Nair2015,
	title = {Reflection modeling for passive stereo},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2015 Inter},
	year = {2015},
	pages = {2291{\textendash}2299},
	abstract = {Stereo reconstruction in presence of reality faces many challenges that still need to be addressed. This paper considers reflections, which introduce incorrect matches due to the observation violating the diffuse-world assumption underlying the majority of stereo techniques. Unlike most existing work, which employ regularization or robust data terms to suppress such errors, we derive two least squares models from first principles that generalize diffuse world stereo and explicitly take reflections into account. These models are parametrized by depth, orientation and material properties, resulting in a total of up to 5 parameters per pixel that have to be estimated. Additionally large non-local interactions between viewed and reflected surface have to be taken into account. These two properties make inference of the model appear prohibitive, but we present evidence that inference is actually possible using a variant of patch match stereo.},
	isbn = {9781467383912},
	issn = {15505499},
	doi = {10.1109/ICCV.2015.264},
	author = {Nair, Rahul and Fitzgibbon, Andrew and Kondermann, Daniel and Carsten Rother}
}
@conference {Zouhar2015,
	title = {Semantic 3-D labeling of ear implants using a global parametric transition prior},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9350},
	year = {2015},
	pages = {177{\textendash}184},
	abstract = {In this work we consider the problem of sematic part-labeling of 3-D meshesof ear implants. This is a challenging problem and automatic solutions are of high practical relevance, since they help to automate the design of hearing aids. The contribution of this work is a new framework which outperforms existing approaches for this task. To achieve the boost in performance we introduce the new concept of a global parametric transition prior. To our knowledge, this is the first time that such a generic prior is used for 3-D mesh processing, and it may be found useful for a large class of 3-D meshes. To foster more research on the important topic of ear implant labeling, we collected a large data set of 3-D meshes, with associated ground truth labels, which we will make publicly available.},
	isbn = {9783319245706},
	issn = {16113349},
	doi = {10.1007/978-3-319-24571-3_22},
	author = {Zouhar, Alexander and Carsten Rother and Fuchs, Siegfried}
}
@incollection {Richmond2015,
	title = {Uncertainty-driven forest predictors for vertebra localization and segmentation},
	volume = {9349},
	year = {2015},
	pages = {653{\textendash}660},
	abstract = {Accurate localization, identification and segmentation of vertebrae is an important task in medical and biological image analysis. The prevailing approach to solve such a task is to first generate pixelindependent features for each vertebra, e.g. via a random forest predictor, which are then fed into an MRF-based objective to infer the optimal MAP solution of a constellation model. We abandon this static, twostage approach and mix feature generation with model-based inference in a new, more flexible, way. We evaluate our method on two data sets with different objectives. The first is semantic segmentation of a 21-part body plan of zebrafish embryos in microscopy images, and the second is localization and identification of vertebrae in benchmark human CT.},
	issn = {16113349},
	doi = {10.1007/978-3-319-24553-9_80},
	author = {Richmond, David and Kainmueller, Dagmar and Glocker, Ben and Carsten Rother and Myers, Gene}
}
@conference {Kainmueller2014,
	title = {Active graph matching for automatic joint segmentation and annotation of C. elegans},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {8673 LNCS},
	number = {PART 1},
	year = {2014},
	pages = {81{\textendash}88},
	abstract = {In this work we present a novel technique we term active graph matching, which integrates the popular active shape model into a sparse graph matching problem. This way we are able to combine the benefits of a global, statistical deformation model with the benefits of a local deformation model in form of a second-order random field. We present a new iterative energy minimization technique which achieves empirically good results. This enables us to exceed state-of-the art results for the task of annotating nuclei in 3D microscopic images of C. elegans. Furthermore with the help of the generalized Hough transform we are able to jointly segment and annotate a large set of nuclei in a fully automatic fashion for the first time. {\textcopyright} 2014 Springer International Publishing.},
	isbn = {9783319104034},
	issn = {16113349},
	doi = {10.1007/978-3-319-10404-1_11},
	author = {Kainmueller, Dagmar and Jug, Florian and Carsten Rother and Myers, Gene}
}
@article {kappes_14_comparative,
	title = {A Comparative Study of Modern Inference Techniques for Structured
Discrete Energy Minimization Problems},
	journal = {CoRR},
	year = {2014},
	note = {1},
	url = {http://arxiv.org/abs/1404.0533},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, S. and Dhruv Batra and Kim, S. and Bernhard X. Kausler and Thorben Kr{\"o}ger and Lellmann, J. and Komodakis, N. and Savchynskyy, B. and Carsten Rother}
}
@article {kappes-1014-bench-arxiv,
	title = {A Comparative Study of Modern Inference Techniques for Structured Discrete Energy Minimization Problems},
	journal = {CoRR},
	volume = {abs/1404.0533},
	year = {2014},
	url = {http://hci.iwr.uni-heidelberg.de/opengm2/},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, Sebastian and Dhruv Batra and Kim, Sungwoong and Bernhard X. Kausler and Thorben Kr{\"o}ger and Lellmann, Jan and Komodakis, Nikos and Savchynskyy, Bogdan and Carsten Rother}
}
@conference {Zheng2014,
	title = {Dense semantic image segmentation with objects and attributes},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2014},
	pages = {3214{\textendash}3221},
	abstract = {The concepts of objects and attributes are both important for describing images precisely, since verbal descriptions often contain both adjectives and nouns (e.g. {\textquoteright}I see a shiny red chair{\textquoteright}). In this paper, we formulate the problem of joint visual attribute and object class image segmentation as a dense multi-labelling problem, where each pixel in an image can be associated with both an object-class and a set of visual attributes labels. In order to learn the label correlations, we adopt a boosting-based piecewise training approach with respect to the visual appearance and co-occurrence cues. We use a filtering-based mean-field approximation approach for efficient joint inference. Further, we develop a hierarchical model to incorporate region-level object and attribute information. Experiments on the aPASCAL, CORE and attribute augmented NYU indoor scenes datasets show that the proposed approach is able to achieve state-of-the-art results.},
	keywords = {Attributes, Image segmentation, Object recognition, Scene Understanding},
	isbn = {9781479951178},
	issn = {10636919},
	doi = {10.1109/CVPR.2014.411},
	url = {http://www.robots.ox.ac.uk/\~{}tvg/http://tu-dresden.de/inf/cvld},
	author = {Zheng, Shuai and Cheng, Ming Ming and Warrell, Jonathan and Sturgess, Paul and Vineet, Vibhav and Carsten Rother and Torr, Philip H.S.}
}
@conference {Hornacek2014,
	title = {Highly overparameterized optical flow using PatchMatch belief propagation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {8691 LNCS},
	number = {PART 3},
	year = {2014},
	pages = {220{\textendash}234},
	abstract = {Motion in the image plane is ultimately a function of 3D motion in space. We propose to compute optical flow using what is ostensibly an extreme overparameterization: depth, surface normal, and frame-to-frame 3D rigid body motion at every pixel, giving a total of 9 DoF. The advantages of such an overparameterization are twofold: first, geometrically meaningful reasoning can be called upon in the optimization, reflecting possible 3D motion in the underlying scene; second, the {\textquoteright}fronto-parallel{\textquoteright} assumption implicit in the use of traditional matching pixel windows is ameliorated because the parameterization determines a plane-induced homography at every pixel. We show that optimization over this high-dimensional, continuous state space can be carried out using an adaptation of the recently introduced PatchMatch Belief Propagation (PMBP) energy minimization algorithm, and that the resulting flow fields compare favorably to the state of the art on a number of small- and large-displacement datasets. {\textcopyright} 2014 Springer International Publishing.},
	keywords = {9 DoF, large displacement, optical flow, PatchMatch, PMBP},
	isbn = {9783319105772},
	issn = {16113349},
	doi = {10.1007/978-3-319-10578-9_15},
	author = {Horn{\'a}{\v c}ek, Michael and Besse, Frederic and Kautz, Jan and Fitzgibbon, Andrew and Carsten Rother}
}
@conference {Hoai2014,
	title = {Learning discriminative localization from weakly labeled data},
	booktitle = {Pattern Recognition},
	volume = {47},
	number = {3},
	year = {2014},
	month = {mar},
	pages = {1523{\textendash}1534},
	abstract = {Visual categorization problems, such as object classification or action recognition, are increasingly often approached using a detection strategy: a classifier function is first applied to candidate subwindows of the image or the video, and then the maximum classifier score is used for class decision. Traditionally, the subwindow classifiers are trained on a large collection of examples manually annotated with masks or bounding boxes. The reliance on time-consuming human labeling effectively limits the application of these methods to problems involving very few categories. Furthermore, the human selection of the masks introduces arbitrary biases (e.g., in terms of window size and location) which may be suboptimal for classification. We propose a novel method for learning a discriminative subwindow classifier from examples annotated with binary labels indicating the presence of an object or action of interest, but not its location. During training, our approach simultaneously localizes the instances of the positive class and learns a subwindow SVM to recognize them. We extend our method to classification of time series by presenting an algorithm that localizes the most discriminative set of temporal segments in the signal. We evaluate our approach on several datasets for object and action recognition and show that it achieves results similar and in many cases superior to those obtained with full supervision. {\textcopyright} 2013 Elsevier Ltd. All rights reserved.},
	keywords = {Discriminative discovery, Event detection, Image classification, Object detection, Time series classification, Weakly supervised learning},
	issn = {00313203},
	doi = {10.1016/j.patcog.2013.09.028},
	author = {Hoai, Minh and Torresani, Lorenzo and De La Torre, Fernando and Carsten Rother}
}
@conference {Marquez-Neila2014,
	title = {Non-parametric higher-order random fields for image segmentation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {8694 LNCS},
	number = {PART 6},
	year = {2014},
	pages = {269{\textendash}284},
	abstract = {Models defined using higher-order potentials are becoming increasingly popular in computer vision. However, the exact representation of a general higher-order potential defined over many variables is computationally unfeasible. This has led prior works to adopt parametric potentials that can be compactly represented. This paper proposes a non-parametric higher-order model for image labeling problems that uses a patch-based representation of its potentials. We use the transformation scheme of [11, 25] to convert the higher-order potentials to a pair-wise form that can be handled using traditional inference algorithms. This representation is able to capture structure, geometrical and topological information of labels from training data and to provide more precise segmentations. Other tasks such as image denoising and reconstruction are also possible. We evaluate our method on denoising and segmentation problems with synthetic and real images. {\textcopyright} 2014 Springer International Publishing.},
	keywords = {biomedical image analysis, higher-order models, image denoising, Image segmentation, random fields, structured prediction},
	isbn = {9783319105987},
	issn = {16113349},
	doi = {10.1007/978-3-319-10599-4_18},
	author = {M{\'a}rquez-Neila, Pablo and Kohli, Pushmeet and Carsten Rother and Baumela, Luis}
}
@article {Jug2014,
	title = {Optimal joint segmentation and tracking of escherichia coli in the mother machine},
	journal = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {8677},
	year = {2014},
	pages = {25{\textendash}36},
	abstract = {We introduce a graphical model for the joint segmentation and tracking of E. coli cells from time lapse videos. In our setup cells are grown in narrow columns (growth channels) in a so-called {\textquotedblleft}Mother Machine{\textquotedblright} [1]. In these growth channels, cells are vertically aligned, grow and divide over time, and eventually leave the channel at the top. The model is built on a large set of cell segmentation hypotheses for each video frame that we extract from data using a novel parametric max-flow variation. Possible tracking assignments between segments across time, including cell identity mapping, cell division, and cell exit events are enumerated. Each such assignment is represented as a binary decision variable with unary costs based on image and object features of the involved segments. We find a cost-minimal and consistent solution by solving an integer linear program. We introduce a new and important type of constraint that ensures that cells exit the Mother Machine in the correct order. Our method finds a globally optimal tracking solution with an accuracy of > 95\% (1.22 times the inter-observer error) and is on average 2 - 11 times faster than the microscope produces the raw data.},
	issn = {16113349},
	doi = {10.1007/978-3-319-12289-2_3},
	author = {Jug, Florian and Pietzsch, Tobias and Kainm{\"u}ller, Dagmar and Funke, Jan and Kaiser, Matthias and van Nimwegen, Erik and Carsten Rother and Myers, Gene}
}
@article {Besse2014b,
	title = {PMBP: PatchMatch Belief Propagation for correspondence field estimation},
	journal = {International Journal of Computer Vision},
	volume = {110},
	number = {1},
	year = {2014},
	pages = {2{\textendash}13},
	abstract = {PatchMatch (PM) is a simple, yet very powerful and successful method for optimizing continuous labelling problems. The algorithm has two main ingredients: the update of the solution space by sampling and the use of the spatial neighbourhood to propagate samples. We show how these ingredients are related to steps in a specific form of belief propagation (BP) in the continuous space, called max-product particle BP (MP-PBP). However, MP-PBP has thus far been too slow to allow complex state spaces. In the case where all nodes share a common state space and the smoothness prior favours equal values, we show that unifying the two approaches yields a new algorithm, PMBP, which is more accurate than PM and orders of magnitude faster than MP-PBP. To illustrate the benefits of our PMBP method we have built a new stereo matching algorithm with unary terms which are borrowed from the recent PM Stereo work and novel realistic pairwise terms that provide smoothness. We have experimentally verified that our method is an improvement over state-of-the-art techniques at sub-pixel accuracy level.},
	keywords = {Belief propagation, Correspondence fields, PatchMatch},
	issn = {15731405},
	doi = {10.1007/s11263-013-0653-9},
	author = {Besse, Frederic and Carsten Rother and Fitzgibbon, Andrew and Kautz, Jan}
}
@article {Besse2014a,
	title = {PMBP: PatchMatch Belief Propagation for correspondence field estimation},
	journal = {International Journal of Computer Vision},
	volume = {110},
	number = {1},
	year = {2014},
	pages = {2{\textendash}13},
	abstract = {PatchMatch (PM) is a simple, yet very powerful and successful method for optimizing continuous labelling problems. The algorithm has two main ingredients: the update of the solution space by sampling and the use of the spatial neighbourhood to propagate samples. We show how these ingredients are related to steps in a specific form of belief propagation (BP) in the continuous space, called max-product particle BP (MP-PBP). However, MP-PBP has thus far been too slow to allow complex state spaces. In the case where all nodes share a common state space and the smoothness prior favours equal values, we show that unifying the two approaches yields a new algorithm, PMBP, which is more accurate than PM and orders of magnitude faster than MP-PBP. To illustrate the benefits of our PMBP method we have built a new stereo matching algorithm with unary terms which are borrowed from the recent PM Stereo work and novel realistic pairwise terms that provide smoothness. We have experimentally verified that our method is an improvement over state-of-the-art techniques at sub-pixel accuracy level.},
	keywords = {Belief propagation, Correspondence fields, PatchMatch},
	issn = {15731405},
	doi = {10.1007/s11263-013-0653-9},
	author = {Besse, Frederic and Carsten Rother and Fitzgibbon, Andrew and Kautz, Jan}
}
@article {Besse2014,
	title = {PMBP: PatchMatch Belief Propagation for correspondence field estimation},
	journal = {International Journal of Computer Vision},
	volume = {110},
	number = {1},
	year = {2014},
	month = {oct},
	pages = {2{\textendash}13},
	publisher = {Kluwer Academic Publishers},
	abstract = {PatchMatch (PM) is a simple, yet very powerful and successful method for optimizing continuous labelling problems. The algorithm has two main ingredients: the update of the solution space by sampling and the use of the spatial neighbourhood to propagate samples. We show how these ingredients are related to steps in a specific form of belief propagation (BP) in the continuous space, called max-product particle BP (MP-PBP). However, MP-PBP has thus far been too slow to allow complex state spaces. In the case where all nodes share a common state space and the smoothness prior favours equal values, we show that unifying the two approaches yields a new algorithm, PMBP, which is more accurate than PM and orders of magnitude faster than MP-PBP. To illustrate the benefits of our PMBP method we have built a new stereo matching algorithm with unary terms which are borrowed from the recent PM Stereo work and novel realistic pairwise terms that provide smoothness. We have experimentally verified that our method is an improvement over state-of-the-art techniques at sub-pixel accuracy level.},
	keywords = {Belief propagation, Correspondence fields, PatchMatch},
	issn = {15731405},
	doi = {10.1007/s11263-013-0653-9},
	author = {Besse, Frederic and Carsten Rother and Fitzgibbon, Andrew and Kautz, Jan}
}
@conference {Hornacek2014a,
	title = {SphereFlow: 6 DoF scene flow from RGB-D pairs},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2014},
	pages = {3526{\textendash}3533},
	abstract = {We take a new approach to computing dense scene flow between a pair of consecutive RGB-D frames. We exploit the availability of depth data by seeking correspondences with respect to patches specified not as the pixels inside square windows, but as the 3D points that are the inliers of spheres in world space. Our primary contribution is to show that by reasoning in terms of such patches under 6 DoF rigid body motions in 3D, we succeed in obtaining compelling results at displacements large and small without relying on either of two simplifying assumptions that pervade much of the earlier literature: brightness constancy or local surface planarity. As a consequence of our approach, our output is a dense field of 3D rigid body motions, in contrast to the 3D translations that are the norm in scene flow. Reasoning in our manner additionally allows us to carry out occlusion handling using a 6 DoF consistency check for the flow computed in both directions and a patchwise silhouette check to help reason about alignments in occlusion areas, and to promote smoothness of the flow fields using an intuitive local rigidity prior. We carry out our optimization in two steps, obtaining a first correspondence field using an adaptation of PatchMatch, and subsequently using alpha-expansion to jointly handle occlusions and perform regularization. We show attractive flow results on challenging synthetic and real-world scenes that push the practical limits of the aforementioned assumptions.},
	isbn = {9781479951178},
	issn = {10636919},
	doi = {10.1109/CVPR.2014.451},
	author = {Horn{\'a}{\v c}ek, Michael and Fitzgibbon, Andrew and Carsten Rother}
}
@conference {Sindeev2013,
	title = {Alpha-flow for video matting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7726 LNCS},
	number = {PART 3},
	year = {2013},
	pages = {438{\textendash}452},
	abstract = {This work addresses the problem of video matting, that is extracting the opacity-layer of a foreground object from a video sequence. We introduce the notion of alpha-flow which corresponds to the flow in the opacity layer. The idea is derived from the process of rotoscoping, where a user-supplied object mask is smoothly interpolated between keyframes while preserving its correspondence with the underlying image. Our key contribution is an algorithm which infers both the opacity masks and the alpha-flow in an efficient and unified manner. We embed our algorithm in an interactive video matting system where the first and last frame of a sequence are given as keyframes, and additional user strokes may be provided in intermediate frames. We show high quality results on various challenging sequences, and give a detailed comparison to competing techniques. {\textcopyright} 2013 Springer-Verlag.},
	isbn = {9783642374302},
	issn = {03029743},
	doi = {10.1007/978-3-642-37431-9_34},
	author = {Sindeev, Mikhail and Konushin, Anton and Carsten Rother}
}
@conference {kappes_13_comparative,
	title = {A Comparative Study of Modern Inference Techniques for Discrete Energy Minimization Problems},
	booktitle = {CVPR 2013. Proceedings},
	year = {2013},
	note = {1},
	doi = {10.1109/CVPR.2013.175},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, S. and Dhruv Batra and Sungwoong, K. and Bernhard X. Kausler and Lellmann, J. and Komodakis, N. and Carsten Rother}
}
@conference {Kappes-2013-benchmark,
	title = {A Comparative Study of Modern Inference Techniques for Discrete Energy Minimization Problem},
	booktitle = {CVPR},
	year = {2013},
	author = {J{\"o}rg H. Kappes and Bj{\"o}rn Andres and Fred A. Hamprecht and Christoph Schn{\"o}rr and Nowozin, Sebastian and Dhruv Batra and Kim, Sungwoong and Bernhard X. Kausler and Lellmann, Jan and Komodakis, Nikos and Carsten Rother}
}
@conference {Hornacek2013,
	title = {Depth super resolution by rigid body self-similarity in 3D},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2013},
	pages = {1123{\textendash}1130},
	abstract = {We tackle the problem of jointly increasing the spatial resolution and apparent measurement accuracy of an input low-resolution, noisy, and perhaps heavily quantized depth map. In stark contrast to earlier work, we make no use of ancillary data like a color image at the target resolution, multiple aligned depth maps, or a database of high-resolution depth exemplars. Instead, we proceed by identifying and merging patch correspondences within the input depth map itself, exploiting patch wise scene self-similarity across depth such as repetition of geometric primitives or object symmetry. While the notion of {\textquoteright}single-image{\textquoteright} super resolution has successfully been applied in the context of color and intensity images, we are to our knowledge the first to present a tailored analogue for depth images. Rather than reason in terms of patches of 2D pixels as others have before us, our key contribution is to proceed by reasoning in terms of patches of 3D points, with matched patch pairs related by a respective 6 DoF rigid body motion in 3D. In support of obtaining a dense correspondence field in reasonable time, we introduce a new 3D variant of Patch Match. A third contribution is a simple, yet effective patch up scaling and merging technique, which predicts sharp object boundaries at the target resolution. We show that our results are highly competitive with those of alternative techniques leveraging even a color image at the target resolution or a database of high-resolution depth exemplars. {\textcopyright} 2013 IEEE.},
	keywords = {dense matching, depth super resolution, optimization},
	issn = {10636919},
	doi = {10.1109/CVPR.2013.149},
	author = {Horn{\'a}{\v c}ek, Michael and Rhemann, Christoph and Gelautz, Margrit and Carsten Rother}
}
@techreport {Schmidt,
	title = {Discriminative Non-blind Deblurring},
	year = {2013},
	abstract = {Non-blind deblurring is an integral component of blind approaches for removing image blur due to camera shake. Even though learning-based deblurring methods exist, they have been limited to the generative case and are compu-tationally expensive. To this date, manually-defined models are thus most widely used, though limiting the attained restoration quality. We address this gap by proposing a dis-criminative approach for non-blind deblurring. One key challenge is that the blur kernel in use at test time is not known in advance. To address this, we analyze existing approaches that use half-quadratic regularization. From this analysis, we derive a discriminative model cascade for image deblurring. Our cascade model consists of a Gaus-sian CRF at each stage, based on the recently introduced regression tree fields. We train our model by loss minimization and use synthetically generated blur kernels to generate training data. Our experiments show that the proposed approach is efficient and yields state-of-the-art restoration quality on images corrupted with synthetic and real blur.},
	author = {Schmidt, Uwe and Carsten Rother and Nowozin, Sebastian and Jancsary, Jeremy and Roth, Stefan}
}
@article {Hosni2013,
	title = {Fast cost-volume filtering for visual correspondence and beyond},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {35},
	number = {2},
	year = {2013},
	pages = {504{\textendash}511},
	abstract = {Many computer vision tasks can be formulated as labeling problems. The desired solution is often a spatially smooth labeling where label transitions are aligned with color edges of the input image. We show that such solutions can be efficiently achieved by smoothing the label costs with a very fast edge-preserving filter. In this paper, we propose a generic and simple framework comprising three steps: 1) constructing a cost volume, 2) fast cost volume filtering, and 3) Winner-Takes-All label selection. Our main contribution is to show that with such a simple framework state-of-the-art results can be achieved for several computer vision applications. In particular, we achieve 1) disparity maps in real time whose quality exceeds those of all other fast (local) approaches on the Middlebury stereo benchmark, and 2) optical flow fields which contain very fine structures as well as large displacements. To demonstrate robustness, the few parameters of our framework are set to nearly identical values for both applications. Also, competitive results for interactive image segmentation are presented. With this work, we hope to inspire other researchers to leverage this framework to other application areas. {\textcopyright} 2012 IEEE.},
	keywords = {Interactive Image Segmentation, optical flow, Stereo Matching},
	issn = {01628828},
	doi = {10.1109/TPAMI.2012.156},
	author = {Hosni, Asmaa and Rhemann, Christoph and Bleyer, Michael and Carsten Rother and Gelautz, Margrit}
}
@conference {Vineet2013,
	title = {Higher order priors for joint intrinsic image, objects, and attributes estimation},
	booktitle = {Advances in Neural Information Processing Systems},
	year = {2013},
	abstract = {Many methods have been proposed to solve the problems of recovering intrinsic scene properties such as shape, reflectance and illumination from a single image, and object class segmentation separately. While these two problems are mutually informative, in the past not many papers have addressed this topic. In this work we explore such joint estimation of intrinsic scene properties recovered from an image, together with the estimation of the objects and attributes present in the scene. In this way, our unified framework is able to capture the correlations between intrinsic properties (reflectance, shape, illumination), objects (table, tv-monitor), and materials (wooden, plastic) in a given scene. For example, our model is able to enforce the condition that if a set of pixels take same object label, e.g. table, most likely those pixels would receive similar reflectance values. We cast the problem in an energy minimization framework and demonstrate the qualitative and quantitative improvement in the overall accuracy on the NYU and Pascal datasets.},
	issn = {10495258},
	author = {Vineet, Vibhav and Carsten Rother and Torr, Philip H.S.}
}
@conference {Jancsary2013,
	title = {Learning convex QP relaxations for structured prediction},
	booktitle = {30th International Conference on Machine Learning, ICML 2013},
	number = {PART 3},
	year = {2013},
	pages = {1952{\textendash}1960},
	abstract = {We introduce a new large margin approach to discriminative training of intractable discrete graphical models. Our approach builds on a convex quadratic programming relaxation of the MAP inference problem. The model parameters are trained directly within this restricted class of energy functions so as to optimize the predictions on the training data. We address the issue of how to parameterize the resulting model and point out its relation to existing approaches. The primary motivation behind our use of the QP relaxation is its computational efficiency; yet, empirically, its predictive accuracy compares favorably to more expensive approaches. This makes it an appealing choice for many practical tasks. Copyright 2013 by the author(s).},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Carsten Rother}
}
@article {Lempitsky2012,
	title = {Branch-and-mincut: Global optimization for image segmentation with high-level priors},
	journal = {Journal of Mathematical Imaging and Vision},
	volume = {44},
	number = {3},
	year = {2012},
	pages = {315{\textendash}329},
	abstract = {Efficient global optimization techniques such as graph cut exist for energies corresponding to binary image segmentation from low-level cues. However, introducing a high-level prior such as a shape prior or a color-distribution prior into the segmentation process typically results in an energy that is much harder to optimize. The main contribution of the paper is a new global optimization framework for a wide class of such energies. The framework is built upon two powerful techniques: graph cut and branch-and-bound. These techniques are unified through the derivation of lower bounds on the energies. Being computable via graph cut, these bounds are used to prune branches within a branchand-bound search. We demonstrate that the new framework can compute globally optimal segmentations for a variety of segmentation scenarios in a reasonable time on a modern CPU. These scenarios include unsupervised segmentation of an object undergoing 3D pose change, category-specific shape segmentation, and the segmentation under intensity/color priors defined by Chan-Vese and GrabCut functionals. {\textcopyright} Springer Science+Business Media, LLC 2011.},
	keywords = {Branch-and-bound, Global optimization, Graph Cuts, Image segmentation, Shape priors},
	issn = {09249907},
	doi = {10.1007/s10851-012-0328-0},
	author = {Lempitsky, Victor and Blake, Andrew and Carsten Rother}
}
@conference {Shekhovtsov2012b,
	title = {Curvature prior for MRF-based segmentation and shape inpainting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7476 LNCS},
	year = {2012},
	month = {sep},
	pages = {41{\textendash}51},
	abstract = {Most image labeling problems such as segmentation and image reconstruction are fundamentally ill-posed and suffer from ambiguities and noise. Higher-order image priors encode high-level structural dependencies between pixels and are key to overcoming these problems. However, in general these priors lead to computationally intractable models. This paper addresses the problem of discovering compact representations of higher-order priors which allow efficient inference. We propose a framework for solving this problem that uses a recently proposed representation of higher-order functions which are encoded as lower envelopes of linear functions. Maximum a Posterior inference on our learned models reduces to minimizing a pairwise function of discrete variables. We show that our framework can learn a compact representation that approximates a low curvature shape prior and demonstrate its effectiveness in solving shape inpainting and image segmentation problems. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642327162},
	issn = {03029743},
	doi = {10.1007/978-3-642-32717-9_5},
	url = {http://arxiv.org/abs/1109.1480},
	author = {Shekhovtsov, Alexander and Kohli, Pushmeet and Carsten Rother}
}
@conference {Shekhovtsov2012a,
	title = {Curvature prior for MRF-based segmentation and shape inpainting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7476 LNCS},
	year = {2012},
	pages = {41{\textendash}51},
	abstract = {Most image labeling problems such as segmentation and image reconstruction are fundamentally ill-posed and suffer from ambiguities and noise. Higher-order image priors encode high-level structural dependencies between pixels and are key to overcoming these problems. However, in general these priors lead to computationally intractable models. This paper addresses the problem of discovering compact representations of higher-order priors which allow efficient inference. We propose a framework for solving this problem that uses a recently proposed representation of higher-order functions which are encoded as lower envelopes of linear functions. Maximum a Posterior inference on our learned models reduces to minimizing a pairwise function of discrete variables. We show that our framework can learn a compact representation that approximates a low curvature shape prior and demonstrate its effectiveness in solving shape inpainting and image segmentation problems. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642327162},
	issn = {03029743},
	doi = {10.1007/978-3-642-32717-9_5},
	url = {www.research.microsoft.com/vision/cambridge http://www.cs.ucl.ac.uk/staff/V.Kolmogorov/papers/StereoSegmentation_PAMI06.pdf\%5Cnpapers3://publication/uuid/F008E9F4-510D-4478-A3C0-1BFB22F6AEA0},
	author = {Shekhovtsov, Alexander and Kohli, Pushmeet and Carsten Rother}
}
@conference {Shekhovtsov2012,
	title = {Curvature prior for MRF-based segmentation and shape inpainting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7476 LNCS},
	year = {2012},
	pages = {41{\textendash}51},
	abstract = {Most image labeling problems such as segmentation and image reconstruction are fundamentally ill-posed and suffer from ambiguities and noise. Higher-order image priors encode high-level structural dependencies between pixels and are key to overcoming these problems. However, in general these priors lead to computationally intractable models. This paper addresses the problem of discovering compact representations of higher-order priors which allow efficient inference. We propose a framework for solving this problem that uses a recently proposed representation of higher-order functions which are encoded as lower envelopes of linear functions. Maximum a Posterior inference on our learned models reduces to minimizing a pairwise function of discrete variables. We show that our framework can learn a compact representation that approximates a low curvature shape prior and demonstrate its effectiveness in solving shape inpainting and image segmentation problems. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642327162},
	issn = {03029743},
	doi = {10.1007/978-3-642-32717-9_5},
	author = {Shekhovtsov, Alexander and Kohli, Pushmeet and Carsten Rother}
}
@conference {Bleyer2012a,
	title = {Extracting 3D scene-consistent object proposals and depth from stereo images},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7576 LNCS},
	number = {PART 5},
	year = {2012},
	pages = {467{\textendash}481},
	abstract = {This work combines two active areas of research in computer vision: unsupervised object extraction from a single image, and depth estimation from a stereo image pair. A recent, successful trend in unsupervised object extraction is to exploit so-called "3D scene-consistency", that is enforcing that objects obey underlying physical constraints of the 3D scene, such as occupancy of 3D space and gravity of objects. Our main contribution is to introduce the concept of 3D scene-consistency into stereo matching. We show that this concept is beneficial for both tasks, object extraction and depth estimation. In particular, we demonstrate that our approach is able to create a large set of 3D scene-consistent object proposals, by varying e.g. the prior on the number of objects. After automatically ranking the proposals we show experimentally that our results are considerably closer to ground truth than state-of-the-art techniques which either use stereo or monocular images. We envision that our method will build the front-end of a future object recognition system for stereo images. {\textcopyright} 2012 Springer-Verlag.},
	keywords = {Object Segmentation, Stereo Matching},
	isbn = {9783642337147},
	issn = {03029743},
	doi = {10.1007/978-3-642-33715-4_34},
	author = {Bleyer, Michael and Rhemann, Christoph and Carsten Rother}
}
@conference {Bleyer2012,
	title = {Extracting 3D scene-consistent object proposals and depth from stereo images},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7576 LNCS},
	number = {PART 5},
	year = {2012},
	pages = {467{\textendash}481},
	abstract = {This work combines two active areas of research in computer vision: unsupervised object extraction from a single image, and depth estimation from a stereo image pair. A recent, successful trend in unsupervised object extraction is to exploit so-called "3D scene-consistency", that is enforcing that objects obey underlying physical constraints of the 3D scene, such as occupancy of 3D space and gravity of objects. Our main contribution is to introduce the concept of 3D scene-consistency into stereo matching. We show that this concept is beneficial for both tasks, object extraction and depth estimation. In particular, we demonstrate that our approach is able to create a large set of 3D scene-consistent object proposals, by varying e.g. the prior on the number of objects. After automatically ranking the proposals we show experimentally that our results are considerably closer to ground truth than state-of-the-art techniques which either use stereo or monocular images. We envision that our method will build the front-end of a future object recognition system for stereo images. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642337147},
	issn = {03029743},
	doi = {10.1007/978-3-642-33715-4_34},
	url = {http://vision.middlebury.edu/stereo/},
	author = {Bleyer, Michael and Rhemann, Christoph and Carsten Rother}
}
@conference {Jancsary2012a,
	title = {Loss-specific training of non-parametric image restoration models: A new state of the art},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7578 LNCS},
	number = {PART 7},
	year = {2012},
	pages = {112{\textendash}125},
	abstract = {After a decade of rapid progress in image denoising, recent methods seem to have reached a performance limit. Nonetheless, we find that state-of-the-art denoising methods are visually clearly distinguishable and possess complementary strengths and failure modes. Motivated by this observation, we introduce a powerful non-parametric image restoration framework based on Regression Tree Fields (RTF). Our restoration model is a densely-connected tractable conditional random field that leverages existing methods to produce an image-dependent, globally consistent prediction. We estimate the conditional structure and parameters of our model from training data so as to directly optimize for popular performance measures. In terms of peak signal-to-noise-ratio (PSNR), our model improves on the best published denoising method by at least 0.26dB across a range of noise levels. Our most practical variant still yields statistically significant improvements, yet is over 20x faster than the strongest competitor. Our approach is well-suited for many more image restoration and low-level vision problems, as evidenced by substantial gains in tasks such as removal of JPEG blocking artefacts. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642337857},
	issn = {03029743},
	doi = {10.1007/978-3-642-33786-4_9},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Carsten Rother}
}
@conference {Jancsary2012,
	title = {Loss-specific training of non-parametric image restoration models: A new state of the art},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {7578 LNCS},
	number = {PART 7},
	year = {2012},
	pages = {112{\textendash}125},
	abstract = {After a decade of rapid progress in image denoising, recent methods seem to have reached a performance limit. Nonetheless, we find that state-of-the-art denoising methods are visually clearly distinguishable and possess complementary strengths and failure modes. Motivated by this observation, we introduce a powerful non-parametric image restoration framework based on Regression Tree Fields (RTF). Our restoration model is a densely-connected tractable conditional random field that leverages existing methods to produce an image-dependent, globally consistent prediction. We estimate the conditional structure and parameters of our model from training data so as to directly optimize for popular performance measures. In terms of peak signal-to-noise-ratio (PSNR), our model improves on the best published denoising method by at least 0.26dB across a range of noise levels. Our most practical variant still yields statistically significant improvements, yet is over 20x faster than the strongest competitor. Our approach is well-suited for many more image restoration and low-level vision problems, as evidenced by substantial gains in tasks such as removal of JPEG blocking artefacts. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642337857},
	issn = {03029743},
	doi = {10.1007/978-3-642-33786-4_9},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Carsten Rother}
}
@conference {Jancsary2012b,
	title = {Non-parametric crfs for image labeling},
	booktitle = {NIPS Workshop Modern Nonparametric Methods in Machine Learning},
	number = {2},
	year = {2012},
	pages = {1{\textendash}5},
	abstract = {We introduce a powerful non-parametric image labeling framework, Regression Tree Fields (RTFs), and discuss its application to image restoration. The conditional structure and the parameters of our model are estimated from training data so as to directly optimize for popular performance measures, resulting in excellent predictive performance at low computational cost.},
	url = {http://www.nowozin.net/sebastian/papers/jancsary2012nonparametriccrf.pdf},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Carsten Rother}
}
@conference {Jancsary2012d,
	title = {Regression Tree Fields An efficient, non-parametric approach to image labeling problems},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2012},
	pages = {2376{\textendash}2383},
	abstract = {We introduce Regression Tree Fields (RTFs), a fully conditional random field model for image labeling problems. RTFs gain their expressive power from the use of non-parametric regression trees that specify a tractable Gaussian random field, thereby ensuring globally consistent predictions. Our approach improves on the recently introduced decision tree field (DTF) model [14] in three key ways: (i) RTFs have tractable test-time inference, making efficient optimal predictions feasible and orders of magnitude faster than for DTFs, (ii) RTFs can be applied to both discrete and continuous vector-valued labeling tasks, and (Hi) the entire model, including the structure of the regression trees and energy function parameters, can be efficiently and jointly learned from training data. We demonstrate the expressive power and flexibility of the RTF model on a wide variety of tasks, including inpainting, colorization, denoising, and joint detection and registration. We achieve excellent predictive performance which is on par with, or even surpassing, DTFs on all tasks where a comparison is possible. {\textcopyright} 2012 IEEE.},
	isbn = {9781467312264},
	issn = {10636919},
	doi = {10.1109/CVPR.2012.6247950},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Sharp, Toby and Carsten Rother}
}
@conference {Jancsary2012c,
	title = {Regression Tree Fields An efficient, non-parametric approach to image labeling problems},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2012},
	pages = {2376{\textendash}2383},
	abstract = {We introduce Regression Tree Fields (RTFs), a fully conditional random field model for image labeling problems. RTFs gain their expressive power from the use of non-parametric regression trees that specify a tractable Gaussian random field, thereby ensuring globally consistent predictions. Our approach improves on the recently introduced decision tree field (DTF) model [14] in three key ways: (i) RTFs have tractable test-time inference, making efficient optimal predictions feasible and orders of magnitude faster than for DTFs, (ii) RTFs can be applied to both discrete and continuous vector-valued labeling tasks, and (Hi) the entire model, including the structure of the regression trees and energy function parameters, can be efficiently and jointly learned from training data. We demonstrate the expressive power and flexibility of the RTF model on a wide variety of tasks, including inpainting, colorization, denoising, and joint detection and registration. We achieve excellent predictive performance which is on par with, or even surpassing, DTFs on all tasks where a comparison is possible. {\textcopyright} 2012 IEEE.},
	isbn = {9781467312264},
	issn = {10636919},
	doi = {10.1109/CVPR.2012.6247950},
	author = {Jancsary, Jeremy and Nowozin, Sebastian and Sharp, Toby and Carsten Rother}
}
@article {Kohli2012,
	title = {User-centric learning and evaluation of interactive segmentation systems},
	journal = {International Journal of Computer Vision},
	volume = {100},
	number = {3},
	year = {2012},
	month = {dec},
	pages = {261{\textendash}274},
	abstract = {Many successful applications of computer vision to image or video manipulation are interactive by nature. However, parameters of such systems are often trained neglecting the user. Traditionally, interactive systems have been treated in the same manner as their fully automatic counterparts. Their performance is evaluated by computing the accuracy of their solutions under some fixed set of user interactions. In this paper, we study the problem of evaluating and learning interactive segmentation systems which are extensively used in the real world. The key questions in this context are how to measure (1) the effort associated with a user interaction, and (2) the quality of the segmentation result as perceived by the user. We conduct a user study to analyze user behavior and answer these questions. Using the insights obtained from these experiments, we propose a framework to evaluate and learn interactive segmentation systems which brings the user in the loop. The framework is based on the use of an active robot user-a simulated model of a human user. We show how this approach can be used to evaluate and learn parameters of state-of-the-art interactive segmentation systems. We also show how simulated user models can be integrated into the popular max-margin method for parameter learning and propose an algorithm to solve the resulting optimisation problem. {\textcopyright} 2012 The Author(s).},
	keywords = {Image segmentation, Interactive systems, Learning},
	issn = {09205691},
	doi = {10.1007/s11263-012-0537-4},
	author = {Kohli, Pushmeet and Nickisch, Hannes and Carsten Rother and Rhemann, Christoph}
}
@conference {Mansfield2012,
	title = {Visibility maps for improving seam carving},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6554 LNCS},
	number = {PART 2},
	year = {2012},
	pages = {131{\textendash}144},
	abstract = {In this paper, we present a new, improved seam carving algorithm. Seam carving efficiently removes pixels from an image to produce a retargeted image. It has proved popular with users and has been used as a component in many retargeting algorithms. We introduce the visibility map, a new framework for pixel removing image editing methods. This allows us to cast retargeting as a binary graph labelling problem. We derive a general algorithm which uses seam carving operations for efficient greedy optimization of a well defined energy, and compare this with forward energy seam carving and shift map image editing. We test this method with varying parameters on a large number of images, and present an improved seam carving algorithm which can demonstrably produce better results. We draw general conclusions about pixel removing methods for retargeting and motivate future directions of research. {\textcopyright} 2012 Springer-Verlag.},
	isbn = {9783642357398},
	issn = {03029743},
	doi = {10.1007/978-3-642-35740-4_11},
	url = {http://www.adobe.com/products/photoshop/photoshopextended/features/},
	author = {Mansfield, Alex and Gehler, Peter and Van Gool, Luc and Carsten Rother}
}
@conference {meister_12_when,
	title = {When Can We Use KinectFusion for Ground Truth Acquisition?},
	booktitle = {Workshop on Color-Depth Camera Fusion in Robotics, IEEE International
Conference on Intelligent Robots and Systems},
	year = {2012},
	note = {1},
	author = {Stephan Meister and Izadi, S. and Kohli, P. and H{\"a}mmerle, M. and Carsten Rother and Daniel Kondermann}
}
@conference {Nowozin2011a,
	title = {Decision tree fields},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2011},
	pages = {1668{\textendash}1675},
	abstract = {This paper introduces a new formulation for discrete image labeling tasks, the Decision Tree Field (DTF), that combines and generalizes random forests and conditional random fields (CRF) which have been widely used in computer vision. In a typical CRF model the unary potentials are derived from sophisticated random forest or boosting based classifiers, however, the pairwise potentials are assumed to (1) have a simple parametric form with a pre-specified and fixed dependence on the image data, and (2) to be defined on the basis of a small and fixed neighborhood. In contrast, in DTF, local interactions between multiple variables are determined by means of decision trees evaluated on the image data, allowing the interactions to be adapted to the image content. This results in powerful graphical models which are able to represent complex label structure. Our key technical contribution is to show that the DTF model can be trained efficiently and jointly using a convex approximate likelihood function, enabling us to learn over a million free model parameters. We show experimentally that for applications which have a rich and complex label structure, our model achieves excellent results. {\textcopyright} 2011 IEEE.},
	isbn = {9781457711015},
	doi = {10.1109/ICCV.2011.6126429},
	author = {Nowozin, Sebastian and Carsten Rother and Bagon, Shai and Sharp, Toby and Yao, Bangpeng and Kohli, Pushmeet}
}
@conference {He2011,
	title = {A global sampling method for alpha matting},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2011},
	pages = {2049{\textendash}2056},
	abstract = {Alpha matting refers to the problem of softly extracting the foreground from an image. Given a trimap (specifying known foreground/background and unknown pixels), a straightforward way to compute the alpha value is to sample some known foreground and background colors for each unknown pixel. Existing sampling-based matting methods often collect samples near the unknown pixels only. They fail if good samples cannot be found nearby. In this paper, we propose a global sampling method that uses all samples available in the image. Our global sample set avoids missing good samples. A simple but effective cost function is defined to tackle the ambiguity in the sample selection process. To handle the computational complexity introduced by the large number of samples, we pose the sampling task as a correspondence problem. The correspondence search is efficiently achieved by generalizing a randomized algorithm previously designed for patch matching[3]. A variety of experiments show that our global sampling method produces both visually and quantitatively high-quality matting results. {\textcopyright} 2011 IEEE.},
	isbn = {9781457703942},
	issn = {10636919},
	doi = {10.1109/CVPR.2011.5995495},
	author = {He, Kaiming and Rhemann, Christoph and Carsten Rother and Tang, Xiaoou and Sun, Jian}
}
@conference {Toppe2011,
	title = {Image-based 3D modeling via cheeger sets},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6492 LNCS},
	number = {PART 1},
	year = {2011},
	pages = {53{\textendash}64},
	abstract = {We propose a novel variational formulation for generating 3D models of objects from a single view. Based on a few user scribbles in an image, the algorithm automatically extracts the object silhouette and subsequently determines a 3D volume by minimizing the weighted surface area for a fixed user-specified volume. The respective energy can be efficiently minimized by means of convex relaxation techniques, leading to visually pleasing smooth surfaces within a matter of seconds. In contrast to existing techniques for single-view reconstruction, the proposed method is based on an implicit surface representation and a transparent optimality criterion, assuring high-quality 3D models of arbitrary topology with a minimum of user input. {\textcopyright} 2011 Springer-Verlag Berlin Heidelberg.},
	isbn = {9783642193149},
	issn = {03029743},
	doi = {10.1007/978-3-642-19315-6_5},
	author = {T{\"o}ppe, Eno and Oswald, Martin R and Daniel Cremers and Carsten Rother}
}
@article {Rother2011,
	title = {Interactive foreground extraction using graph cut},
	journal = {Advances in Markov \ldots},
	year = {2011},
	pages = {1{\textendash}20},
	abstract = {Note, this is an extended version of chapter 7 from the book: Markov Random Fields for Vision and Image Processing, MIT Press [6]. In this Technical Report, references to other chapters are with respect to the book. The differences are, a new section 4.3 and extra details in section 3.2 and 3.3 The topic of interactive image segmentation has received considerable attention in the computer vision community in the last decade. Today, this topic is very mature and commercial products exist which feature advanced research solutions. This means that interactive image segmentation is today probably one of the most used computer vision technologies worldwide. In this chapter we review one class of interactive segmen-tation techniques, which use discrete optimization and a regional selection interface. We begin the chapter by explaining the seminal work of Boykov and Jolly [9]. After that the GrabCut technique [36] is introduced, which improves on [9]. GrabCut is the underlying algorithm for the Background Removal tool in the Microsoft Office 2010 product. In the third part of the chapter many interesting features and details are explained which are part of the product. In this process several recent research articles are reviewed. Finally, the Background Removal tool, as well as [9, 36], are evaluated in different ways on publicly available databases. This includes static and dynamic user inputs. 1},
	url = {http://research.microsoft.com/pubs/147408/rotheretalmrfbook-grabcut.pdf},
	author = {Carsten Rother and Kolmogorov, Vladimir}
}
@conference {Vicente2011,
	title = {Object cosegmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2011},
	pages = {2217{\textendash}2224},
	abstract = {Cosegmentation is typically defined as the task of jointly segmenting something similar in a given set of images. Existing methods are too generic and so far have not demonstrated competitive results for any specific task. In this paper we overcome this limitation by adding two new aspects to cosegmentation: (1) the "something" has to be an object, and (2) the "similarity" measure is learned. In this way, we are able to achieve excellent results on the recently introduced iCoseg dataset, which contains small sets of images of either the same object instance or similar objects of the same class. The challenge of this dataset lies in the extreme changes in viewpoint, lighting, and object deformations within each set. We are able to considerably outperform several competitors. To achieve this performance, we borrow recent ideas from object recognition: the use of powerful features extracted from a pool of candidate object-like segmentations. We believe that our work will be beneficial to several application areas, such as image retrieval. {\textcopyright} 2011 IEEE.},
	isbn = {9781457703942},
	issn = {10636919},
	doi = {10.1109/CVPR.2011.5995530},
	author = {Vicente, Sara and Carsten Rother and Kolmogorov, Vladimir}
}
@conference {Bleyer2011,
	title = {Object stereo Joint stereo matching and object segmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2011},
	pages = {3081{\textendash}3088},
	abstract = {This paper presents a method for joint stereo matching and object segmentation. In our approach a 3D scene is represented as a collection of visually distinct and spatially coherent objects. Each object is characterized by three different aspects: a color model, a 3D plane that approximates the object{\textquoteright}s disparity distribution, and a novel 3D connectivity property. Inspired by Markov Random Field models of image segmentation, we employ object-level color models as a soft constraint, which can aid depth estimation in powerful ways. In particular, our method is able to recover the depth of regions that are fully occluded in one input view, which to our knowledge is new for stereo matching. Our model is formulated as an energy function that is optimized via fusion moves. We show high-quality disparity and object segmentation results on challenging image pairs as well as standard benchmarks. We believe our work not only demonstrates a novel synergy between the areas of image segmentation and stereo matching, but may also inspire new work in the domain of automatic and interactive object-level scene manipulation. {\textcopyright} 2011 IEEE.},
	isbn = {9781457703942},
	issn = {10636919},
	doi = {10.1109/CVPR.2011.5995581},
	author = {Bleyer, Michael and Carsten Rother and Kohli, Pushmeet and Scharstein, Daniel and Sinha, Sudipta}
}
@conference {Bleyer2011a,
	title = {PatchMatch Stereo - Stereo Matching with Slanted Support Windows},
	year = {2011},
	pages = {14.1{\textendash}14.11},
	abstract = {Common local stereo methods match support windows at integer-valued disparities. The implicit assumption that pixels within the support region have constant disparity does not hold for slanted surfaces and leads to a bias towards reconstructing frontoparallel surfaces. This work overcomes this bias by estimating an individual 3D plane at each pixel onto which the support region is projected. The major challenge of this approach is to find a pixels optimal 3D plane among all possible planes whose number is infinite. We show that an ideal algorithm to solve this problem is PatchMatch 1 that we extend to find an approximate nearest neighbor according to a plane. In addition to Patch- Matchs spatial propagation scheme, we propose (1) view propagation where planes are propagated among left and right views of the stereo pair and (2) temporal propagation where planes are propagated from preceding and consecutive frames of a video when doing temporal stereo. Adaptive support weights are used in matching cost aggregation to improve results at disparity borders. We also show that our slanted support windows can be used to compute a cost volume for global stereo methods, which allows for explicit treatment of occlusions and can handle large untextured regions. In the results we demonstrate that our method reconstructs highly slanted surfaces and achieves impressive disparity details with sub-pixel precision. In the Middlebury table, our method is currently top-performer among local methods and takes rank 2 among approximately 110 competitors if sub-pixel precision is considered.},
	doi = {10.5244/c.25.14},
	author = {Bleyer, Michael and Rhemann, Christoph and Carsten Rother}
}
@conference {Pletscher2011a,
	title = {Putting MAP back on the map},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6835 LNCS},
	year = {2011},
	pages = {111{\textendash}121},
	abstract = {Conditional Random Fields (CRFs) are popular models in computer vision for solving labeling problems such as image denoising. This paper tackles the rarely addressed but important problem of learning the full form of the potential functions of pairwise CRFs. We examine two popular learning techniques, maximum likelihood estimation and maximum margin training. The main focus of the paper is on models such as pairwise CRFs, that are simplistic (misspecified) and do not fit the data well. We empirically demonstrate that for misspecified models maximum-margin training with MAP prediction is superior to maximum likelihood estimation with any other prediction method. Additionally we examine the common belief that MLE is better at producing predictions matching image statistics. {\textcopyright} 2011 Springer-Verlag.},
	isbn = {9783642231223},
	issn = {03029743},
	doi = {10.1007/978-3-642-23123-0_12},
	author = {Pletscher, Patrick and Nowozin, Sebastian and Kohli, Pushmeet and Carsten Rother}
}
@conference {Pletscher2011,
	title = {Putting MAP back on the map},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6835 LNCS},
	year = {2011},
	pages = {111{\textendash}121},
	abstract = {Conditional Random Fields (CRFs) are popular models in computer vision for solving labeling problems such as image denoising. This paper tackles the rarely addressed but important problem of learning the full form of the potential functions of pairwise CRFs. We examine two popular learning techniques, maximum likelihood estimation and maximum margin training. The main focus of the paper is on models such as pairwise CRFs, that are simplistic (misspecified) and do not fit the data well. We empirically demonstrate that for misspecified models maximum-margin training with MAP prediction is superior to maximum likelihood estimation with any other prediction method. Additionally we examine the common belief that MLE is better at producing predictions matching image statistics. {\textcopyright} 2011 Springer-Verlag.},
	isbn = {9783642231223},
	issn = {03029743},
	doi = {10.1007/978-3-642-23123-0_12},
	author = {Pletscher, Patrick and Nowozin, Sebastian and Kohli, Pushmeet and Carsten Rother}
}
@conference {Gehler2011,
	title = {Recovering intrinsic images with a global sparsity prior on reflectance},
	booktitle = {Advances in Neural Information Processing Systems 24: 25th Annual Conference on Neural Information Processing Systems 2011, NIPS 2011},
	year = {2011},
	abstract = {We address the challenging task of decoupling material properties from lighting properties given a single image. In the last two decades virtually all works have concentrated on exploiting edge information to address this problem. We take a different route by introducing a new prior on reflectance, that models reflectance values as being drawn from a sparse set of basis colors. This results in a Random Field model with global, latent variables (basis colors) and pixel-accurate output reflectance values. We show that without edge information high-quality results can be achieved, that are on par with methods exploiting this source of information. Finally, we are able to improve on state-of-the-art results by integrating edge information into our model. We believe that our new approach is an excellent starting point for future developments in this field.},
	isbn = {9781618395993},
	author = {Gehler, Peter Vincent and Carsten Rother and Kiefel, Martin and Zhang, Lumin and Sch{\"o}lkopf, Bernhard}
}
@article {Rother2011a,
	title = {Sparse Higher Order Functions of Discrete Variables{\textendash}-Representation and Optimization},
	journal = {research.microsoft.com},
	volume = {45},
	year = {2011},
	abstract = {Higher order energy functions have the ability to encode high level structural dependencies between pix-els, which have been shown to be extremely powerful for image labeling problems. Their use, however , is severely hampered in practice by the intractable complexity of representing and minimizing such functions. We observed that higher order functions encountered in computer vision are very often "sparse", i.e. many labelings of a higher order clique are equally unlikely and hence have the same high cost. In this paper, we address the problem of minimizing such sparse higher order energy functions. Our method works by transforming the problem into an equivalent quadratic function minimization problem. The resulting quadratic function can be minimized using popular message passing or graph cut based algorithms for MAP inference. Although this is primarily a theoretical paper, we also show how labeling problems such as texture denoising and inpainting can be formulated using sparse higher order energy functions. We demonstrate experimentally that for some challenging tasks our formulation is able to out-perform various state-of-the art techniques, especially the well-known patch-based approach of Freeman et al. [11]. Given the broad use of patch-based models in computer vision, we believe that our contributions will be applicable in many problem domains.},
	url = {http://research.microsoft.com/pubs/147370/RotherKohli-SparseHigherOrder.pdf},
	author = {Carsten Rother}
}
@conference {Vicente2010,
	title = {Cosegmentation revisited: Models and optimization},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6312 LNCS},
	number = {PART 2},
	year = {2010},
	pages = {465{\textendash}479},
	abstract = {The problem of cosegmentation consists of segmenting the same object (or objects of the same class) in two or more distinct images. Recently a number of different models have been proposed for this problem. However, no comparison of such models and corresponding optimization techniques has been done so far. We analyze three existing models: the L1 norm model of Rother et al. [1], the L2 norm model of Mukherjee et al. [2] and the "reward" model of Hochbaum and Singh [3]. We also study a new model, which is a straightforward extension of the Boykov-Jolly model for single image segmentation [4]. In terms of optimization, we use a Dual Decomposition (DD) technique in addition to optimization methods in [1,2]. Experiments show a significant improvement of DD over published methods. Our main conclusion, however, is that the new model is the best overall because it: (i) has fewest parameters; (ii) is most robust in practice, and (iii) can be optimized well with an efficient EM-style procedure. {\textcopyright} 2010 Springer-Verlag.},
	isbn = {3642155510},
	issn = {16113349},
	doi = {10.1007/978-3-642-15552-9_34},
	author = {Vicente, Sara and Kolmogorov, Vladimir and Carsten Rother}
}
@article {Lempitsky2010a,
	title = {Fusion moves for markov random field optimization},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {32},
	number = {8},
	year = {2010},
	pages = {1392{\textendash}1405},
	abstract = {The efficient application of graph cuts to Markov Random Fields (MRFs) with multiple discrete or continuous labels remains an open question. In this paper, we demonstrate one possible way of achieving this by using graph cuts to combine pairs of suboptimal labelings or solutions. We call this combination process the fusion move. By employing recently developed graph-cut-based algorithms (so-called QPBO-graph cut), the fusion move can efficiently combine two proposal labelings in a theoretically sound way, which is in practice often globally optimal. We demonstrate that fusion moves generalize many previous graph-cut approaches, which allows them to be used as building blocks within a broader variety of optimization schemes than were considered before. In particular, we propose new optimization schemes for computer vision MRFs with applications to image restoration, stereo, and optical flow, among others. Within these schemes the fusion moves are used 1) for the parallelization of MRF optimization into several threads, 2) for fast MRF optimization by combining cheap-to-compute solutions, and 3) for the optimization of highly nonconvex continuous-labeled MRFs with 2D labels. Our final example is a nonvision MRF concerned with cartographic label placement, where fusion moves can be used to improve the performance of a standard inference method (loopy belief propagation). {\textcopyright} 2006 IEEE.},
	keywords = {Combinatorial algorithms, Computer vision, Graph algorithms, Image restoration., Markov random fields, Motion, Stereo},
	issn = {01628828},
	doi = {10.1109/TPAMI.2009.143},
	author = {Lempitsky, Victor and Carsten Rother and Roth, Stefan and Blake, Andrew}
}
@article {Lempitsky2010,
	title = {Fusion moves for markov random field optimization},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {32},
	number = {8},
	year = {2010},
	pages = {1392{\textendash}1405},
	abstract = {The efficient application of graph cuts to Markov Random Fields (MRFs) with multiple discrete or continuous labels remains an open question. In this paper, we demonstrate one possible way of achieving this by using graph cuts to combine pairs of suboptimal labelings or solutions. We call this combination process the fusion move. By employing recently developed graph-cut-based algorithms (so-called QPBO-graph cut), the fusion move can efficiently combine two proposal labelings in a theoretically sound way, which is in practice often globally optimal. We demonstrate that fusion moves generalize many previous graph-cut approaches, which allows them to be used as building blocks within a broader variety of optimization schemes than were considered before. In particular, we propose new optimization schemes for computer vision MRFs with applications to image restoration, stereo, and optical flow, among others. Within these schemes the fusion moves are used 1) for the parallelization of MRF optimization into several threads, 2) for fast MRF optimization by combining cheap-to-compute solutions, and 3) for the optimization of highly nonconvex continuous-labeled MRFs with 2D labels. Our final example is a nonvision MRF concerned with cartographic label placement, where fusion moves can be used to improve the performance of a standard inference method (loopy belief propagation). {\textcopyright} 2006 IEEE.},
	keywords = {Combinatorial algorithms, Computer vision, Graph algorithms, Image restoration., Markov random fields, Motion, Stereo},
	issn = {01628828},
	doi = {10.1109/TPAMI.2009.143},
	author = {Lempitsky, Victor and Carsten Rother and Roth, Stefan and Blake, Andrew}
}
@conference {Gulshan2010,
	title = {Geodesic star convexity for interactive image segmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2010},
	pages = {3129{\textendash}3136},
	abstract = {In this paper we introduce a new shape constraint for interactive image segmentation. It is an extension of Veksler{\textquoteright}s [25] star-convexity prior, in two ways: from a single star to multiple stars and from Euclidean rays to Geodesic paths. Global minima of the energy function are obtained subject to these new constraints. We also introduce Geodesic Forests, which exploit the structure of shortest paths in implementing the extended constraints. The star-convexity prior is used here in an interactive setting and this is demonstrated in a practical system. The system is evaluated by means of a "robot user" to measure the amount of interaction required in a precise way. We also introduce a new and harder dataset which augments the existing Grabcut dataset [1] with images and ground truth taken from the PASCAL VOC segmentation challenge [7]. {\textcopyright}2010 IEEE.},
	isbn = {9781424469840},
	issn = {10636919},
	doi = {10.1109/CVPR.2010.5540073},
	author = {Gulshan, Varun and Carsten Rother and Criminisi, Antonio and Blake, Andrew and Zisserman, Andrew}
}
@conference {Rother2010,
	title = {Minimizing sparse higher order energy functions of discrete variables},
	year = {2010},
	pages = {1382{\textendash}1389},
	abstract = {Higher order energy functions have the ability to encode high level structural dependencies between pixels, which have been shown to be extremely powerful for image labeling problems. Their use, however, is severely hampered in practice by the intractable complexity of representing and minimizing such functions. We observed that higher order functions encountered in computer vision are very often {\textquotedblleft}sparse{\textquotedblright}, i.e. many labelings of a higher order clique are equally unlikely and hence have the same high cost. In this paper, we address the problem of minimizing such sparse higher order energy functions. Our method works by transforming the problem into an equivalent quadratic function minimization problem. The resulting quadratic function can be minimized using popular message passing or graph cut based algorithms for MAP inference. Although this is primarily a theoretical paper, it also shows how higher order functions can be used to obtain impressive results for the binary texture restoration problem.},
	doi = {10.1109/cvpr.2009.5206739},
	author = {Carsten Rother and Kohli, Pushmeet and Wei Feng and Jiaya Jia}
}
@conference {Singaraju2010,
	title = {New appearance models for natural image matting},
	year = {2010},
	pages = {659{\textendash}666},
	abstract = {Image matting is the task of estimating a fore- and background layer from a single image. To solve this ill posed problem, an accurate modeling of the scene{\textquoteright}s appearance is necessary. Existing methods that provide a closed form solution to this problem, assume that the colors of the foreground and background layers are locally linear. In this paper, we show that such models can be an overfit when the colors of the two layers are locally constant. We derive new closed form expressions in such cases, and show that our models are more compact than existing ones. In particular, the null space of our cost function is a subset of the null space constructed by existing approaches. We discuss the bias towards specific solutions for each formulation. Experiments on synthetic and real data confirm that our compact models estimate alpha mattes more accurately than existing techniques, without the need of additional user interaction.},
	doi = {10.1109/cvpr.2009.5206491},
	author = {Singaraju, Dheeraj and Carsten Rother and Rhemann, Christoph}
}
@conference {Mansfield2010,
	title = {Scene carving: Scene consistent image retargeting},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6311 LNCS},
	number = {PART 1},
	year = {2010},
	pages = {143{\textendash}156},
	abstract = {Image retargeting algorithms often create visually disturbing distortion. We introduce the property of scene consistency, which is held by images which contain no object distortion and have the correct object depth ordering. We present two new image retargeting algorithms that preserve scene consistency. These algorithms make use of a user-provided relative depth map, which can be created easily using a simple GrabCut-style interface. Our algorithms generalize seam carving. We decompose the image retargeting procedure into (a) removing image content with minimal distortion and (b) re-arrangement of known objects within the scene to maximize their visibility. Our algorithms optimize objectives (a) and (b) jointly. However, they differ considerably in how they achieve this. We discuss this in detail and present examples illustrating the rationale of preserving scene consistency in retargeting. {\textcopyright} 2010 Springer-Verlag.},
	isbn = {3642155480},
	issn = {16113349},
	doi = {10.1007/978-3-642-15549-9_11},
	url = {www.fujifilm.com/products/3d/camera/finepix_},
	author = {Mansfield, Alex and Gehler, Peter and Van Gool, Luc and Carsten Rother}
}
@conference {Rhemann2010,
	title = {A spatially varying PSF-based prior for alpha matting},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2010},
	pages = {2149{\textendash}2156},
	abstract = {In this paper we considerably improve on a state-of-theart alpha matting approach by incorporating a new prior which is based on the image formation process. In particular, we model the prior probability of an alpha matte as the convolution of a high-resolution binary segmentation with the spatially varying point spread function (PSF) of the camera. Our main contribution is a new and efficient deconvolution approach that recovers the prior model, given an approximate alpha matte. By assuming that the PSF is a kernel with a single peak, we are able to recover the binary segmentation with an MRF-based approach, which exploits flux and a new way of enforcing connectivity. The spatially varying PSF is obtained via a partitioning of the image into regions of similar defocus. Incorporating our new prior model into a state-of-the-art matting technique produces results that outperform all competitors, which we confirm using a publicly available benchmark. {\textcopyright}2010 IEEE.},
	isbn = {9781424469840},
	issn = {10636919},
	doi = {10.1109/CVPR.2010.5539894},
	author = {Rhemann, Christoph and Carsten Rother and Kohli, Pushmeet and Gelautz, Margrit}
}
@conference {Bleyer2010,
	title = {Surface stereo with soft segmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2010},
	pages = {1570{\textendash}1577},
	abstract = {This paper proposes a new stereo model which encodes the simple assumption that the scene is composed of a few, smooth surfaces. A key feature of our model is the surface-based representation, where each pixel is assigned to a 3D surface (planes or B-splines). This representation enables several important contributions: Firstly, we formulate a higher-order prior which states that pixels of similar appearance are likely to belong to the same 3D surface. This enables to incorporate the very popular color segmentation constraint in a soft and principled way. Secondly, we use a global MDL prior to penalize the number of surfaces. Thirdly, we are able to incorporate, in a simple way, a prior which favors low curvature surfaces. Fourthly, we improve the asymmetric occlusion model by disallowing pixels of the same surface to occlude each other. Finally, we use the known fusion move approach which enables a powerful optimization of our model, despite the infinite number of possible labelings (surfaces). {\textcopyright}2010 IEEE.},
	isbn = {9781424469840},
	issn = {10636919},
	doi = {10.1109/CVPR.2010.5539783},
	author = {Bleyer, Michael and Carsten Rother and Kohli, Pushmeet}
}
@conference {Glocker2010,
	title = {TriangleFlow: Optical flow with triangulation-based higher-order likelihoods},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {6313 LNCS},
	number = {PART 3},
	year = {2010},
	pages = {272{\textendash}285},
	abstract = {We use a simple yet powerful higher-order conditional random field (CRF) to model optical flow. It consists of a standard photo-consistency cost and a prior on affine motions both modeled in terms of higher-order potential functions. Reasoning jointly over a large set of unknown variables provides more reliable motion estimates and a robust matching criterion. One of the main contributions is that unlike previous region-based methods, we omit the assumption of constant flow. Instead, we consider local affine warps whose likelihood energy can be computed exactly without approximations. This results in a tractable, so-called, higher-order likelihood function. We realize this idea by employing triangulation meshes which immensely reduce the complexity of the problem. Optimization is performed by hierarchical fusion moves and an adaptive mesh refinement strategy. Experiments show that we achieve high-quality motion fields on several data sets including the Middlebury optical flow database. {\textcopyright} 2010 Springer-Verlag.},
	isbn = {364215557X},
	issn = {16113349},
	doi = {10.1007/978-3-642-15558-1_20},
	url = {http://vision.middlebury.edu/flow/},
	author = {Glocker, Ben and Heibel, T. Hauke and Navab, Nassir and Kohli, Pushmeet and Carsten Rother}
}
@conference {Shesh2009,
	title = {3D-aware image editing for out of bounds photography},
	booktitle = {Proceedings - Graphics Interface},
	year = {2009},
	pages = {47{\textendash}54},
	abstract = {In this paper, we propose algorithms to manipulate 2D images in a way that is consistent with the 3D geometry of the scene that they capture. We present these algorithms in the context of creating "Out of Bounds" (OOB) images - compelling, depth-rich images generated from single, conventional 2D photographs (fig. 1). Starting from a single image our tool enables rapid OOB prototyping; i.e. the ability to quickly create and experiment with many different variants of the OOB effect before deciding which one best expresses the users{\textquoteright} artistic intentions. We achieve this with a flexible work-flow driven by an intuitive user interface. The rich 3D perception of the final composition is achieved by exploiting two strong cues - occlusions and shadows. A realistic-looking 3D frame is interactively inserted in the scene between segmented foreground objects and the background to generate novel occlusions and enhance the scene{\textquoteright}s perception of depth. This perception is further enhanced by adding new, realistic cast shadows. The key contributions of this paper are: (i) new algorithms for inserting simple 3D objects like frames in 2D images requiring minimal camera calibration, and (ii) new techniques for the realistic synthesis of cast shadows, even for complex 3D objects. These algorithms, although presented for OOB photography, may be directly used in general image composition tasks. With our tool, untrained users can turn ordinary photos into compelling OOB images in seconds. In contrast with existing workflows, at any time the artist can modify any aspect of the composition while avoiding time-consuming pixel painting operations. Such a tool has important commercial applications, and is much more suitable for OOB prototyping than existing image editors.},
	isbn = {9781568814704},
	issn = {07135424},
	url = {http://www.flickr.com/groups/oob/},
	author = {Shesh, Amit and Criminisi, Antonio and Carsten Rother and Smyth, Gavin}
}
@conference {Lempitsky2009,
	title = {Image segmentation with a bounding box prior},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2009},
	pages = {277{\textendash}284},
	abstract = {User-provided object bounding box is a simple and popular interaction paradigm considered by many existing interactive image segmentation frameworks. However, these frameworks tend to exploit the provided bounding box merely to exclude its exterior from consideration and sometimes to initialize the energy minimization. In this paper, we discuss how the bounding box can be further used to impose a powerful topological prior, which prevents the solution from excessive shrinking and ensures that the user-provided box bounds the segmentation in a sufficiently tight way. The prior is expressed using hard constraints incorporated into the global energy minimization framework leading to an NP-hard integer program. We then investigate the possible optimization strategies including linear relaxation as well as a new graph cut algorithm called pinpointing. The latter can be used either as a rounding method for the fractional LP solution, which is provably better than thresholding-based rounding, or as a fast standalone heuristic. We evaluate the proposed algorithms on a publicly available dataset, and demonstrate the practical benefits of the new prior both qualitatively and quantitatively. {\textcopyright}2009 IEEE.},
	isbn = {9781424444205},
	doi = {10.1109/ICCV.2009.5459262},
	author = {Lempitsky, Victor and Kohli, Pushmeet and Carsten Rother and Sharp, Toby}
}
@conference {Vicente2009,
	title = {Joint optimization of segmentation and appearance models},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2009},
	pages = {755{\textendash}762},
	abstract = {Many interactive image segmentation approaches use an objective function which includes appearance models as an unknown variable. Since the resulting optimization problem is NP-hard the segmentation and appearance are typically optimized separately, in an EM-style fashion. One contribution of this paper is to express the objective function purely in terms of the unknown segmentation, using higher-order cliques. This formulation reveals an interesting bias of the model towards balanced segmentations. Furthermore, it enables us to develop a new dual decomposition optimization procedure, which provides additionally a lower bound. Hence, we are able to improve on existing optimizers, and verify that for a considerable number of real world examples we even achieve global optimality. This is important since we are able, for the first time, to analyze the deficiencies of the model. Another contribution is to establish a property of a particular dual decomposition approach which involves convex functions depending on foreground area. As a consequence, we show that the optimal decomposition for our problem can be computed efficiently via a parametric maxflow algorithm. {\textcopyright}2009 IEEE.},
	isbn = {9781424444205},
	doi = {10.1109/ICCV.2009.5459287},
	author = {Vicente, Sara and Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Singaraju2009,
	title = {New appearance models for natural image matting},
	booktitle = {2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops 2009},
	volume = {2009 IEEE},
	year = {2009},
	pages = {659{\textendash}666},
	abstract = {Image matting is the task of estimating a fore- and background layer from a single image. To solve this ill posed problem, an accurate modeling of the scene{\textquoteright}s appearance is necessary. Existing methods that provide a closed form solution to this problem, assume that the colors of the foreground and background layers are locally linear. In this paper, we show that such models can be an overfit when the colors of the two layers are locally constant. We derive new closed form expressions in such cases, and show that our models are more compact than existing ones. In particular, the null space of our cost function is a subset of the null space constructed by existing approaches. We discuss the bias towards specific solutions for each formulation. Experiments on synthetic and real data confirm that our compact models estimate alpha mattes more accurately than existing techniques, without the need of additional user interaction. {\textcopyright} 2009 IEEE.},
	isbn = {9781424439935},
	doi = {10.1109/CVPRW.2009.5206491},
	author = {Singaraju, Dheeraj and Carsten Rother and Rhemann, Christoph}
}
@conference {Rhemann2009b,
	title = {A perceptually motivated online benchmark for image matting},
	booktitle = {2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops 2009},
	volume = {2009 IEEE},
	year = {2009},
	pages = {1826{\textendash}1833},
	abstract = {The availability of quantitative online benchmarks for low-level vision tasks such as stereo and optical flow has led to significant progress in the respective fields. This paper introduces such a benchmark for image matting. There are three key factors for a successful benchmarking system: (a) a challenging, high-quality ground truth test set; (b) an online evaluation repository that is dynamically updated with new results; (c) perceptually motivated error functions. Our new benchmark strives to meet all three criteria. We evaluated several matting methods with our benchmark and show that their performance varies depending on the error function. Also, our challenging test set reveals problems of existing algorithms, not reflected in previously reported results. We hope that our effort will lead to considerable progress in the field of image matting, and welcome the reader to visit our benchmark at www.alphamatting.com. {\textcopyright} 2009 IEEE.},
	isbn = {9781424439935},
	doi = {10.1109/CVPRW.2009.5206503},
	author = {Rhemann, Christoph and Carsten Rother and Wang, Jue and Gelautz, Margrit and Kohli, Pushmeet and Rott, Pamela}
}
@conference {Rhemann2009a,
	title = {A perceptually motivated online benchmark for image matting},
	booktitle = {2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops 2009},
	volume = {2009 IEEE},
	year = {2009},
	pages = {1826{\textendash}1833},
	abstract = {The availability of quantitative online benchmarks for low-level vision tasks such as stereo and optical flow has led to significant progress in the respective fields. This paper introduces such a benchmark for image matting. There are three key factors for a successful benchmarking system: (a) a challenging, high-quality ground truth test set; (b) an online evaluation repository that is dynamically updated with new results; (c) perceptually motivated error functions. Our new benchmark strives to meet all three criteria. We evaluated several matting methods with our benchmark and show that their performance varies depending on the error function. Also, our challenging test set reveals problems of existing algorithms, not reflected in previously reported results. We hope that our effort will lead to considerable progress in the field of image matting, and welcome the reader to visit our benchmark at www.alphamatting.com. {\textcopyright} 2009 IEEE.},
	keywords = {Benchmark, Evaluation, Matting},
	isbn = {9781424439935},
	doi = {10.1109/CVPRW.2009.5206503},
	url = {www.alphamatting.com.},
	author = {Rhemann, Christoph and Carsten Rother and Wang, Jue and Gelautz, Margrit and Kohli, Pushmeet and Rott, Pamela}
}
@conference {Rhemann2009,
	title = {A perceptually motivated online benchmark for image matting},
	booktitle = {2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops 2009},
	volume = {2009 IEEE},
	year = {2009},
	pages = {1826{\textendash}1833},
	abstract = {The availability of quantitative online benchmarks for low-level vision tasks such as stereo and optical flow has led to significant progress in the respective fields. This paper introduces such a benchmark for image matting. There are three key factors for a successful benchmarking system: (a) a challenging, high-quality ground truth test set; (b) an online evaluation repository that is dynamically updated with new results; (c) perceptually motivated error functions. Our new benchmark strives to meet all three criteria. We evaluated several matting methods with our benchmark and show that their performance varies depending on the error function. Also, our challenging test set reveals problems of existing algorithms, not reflected in previously reported results. We hope that our effort will lead to considerable progress in the field of image matting, and welcome the reader to visit our benchmark at www.alphamatting.com. {\textcopyright} 2009 IEEE.},
	isbn = {9781424439935},
	doi = {10.1109/CVPRW.2009.5206503},
	url = {www.alphamatting.com.},
	author = {Rhemann, Christoph and Carsten Rother and Wang, Jue and Gelautz, Margrit and Kohli, Pushmeet and Rott, Pamela}
}
@conference {Bleyer2009,
	title = {A stereo approach that handles the matting problem via imagewarping},
	booktitle = {2009 IEEE Computer Society Conference on Computer Vision and Pattern Recognition Workshops, CVPR Workshops 2009},
	volume = {2009 IEEE},
	year = {2009},
	pages = {501{\textendash}508},
	abstract = {We propose an algorithm that simultaneously extracts disparities and alpha matting information given a stereo image pair. Our method divides the reference image into a set of overlapping, partially transparent color segments. Each segment pixel is assigned an alpha value and color. The disparity inside the segment is modeled via a plane. The goodness of alphas, colors and disparity planes is measured by a new energy function. Its basic idea is to use the three parameters for generating artificial views representing the left and right images. If alphas, colors and disparity planes are correct, these artificial images should be very similar to the real ones. For generating the artificial right view, we warp all pixels of the left into the geometry of the right image using the disparity planes. We introduce the assumption of constant solidity in order to correctly model how pixels{\textquoteright} alpha values are affected by the warping operation. Experimental results on the Middlebury set show that our algorithm gives good results in comparison to the state-of-the-art in stereo matching. {\textcopyright}2009 IEEE.},
	keywords = {Alpha Matting, Stereo Matching, Stereo Matting},
	isbn = {9781424439935},
	doi = {10.1109/CVPRW.2009.5206656},
	author = {Bleyer, Michael and Gelautz, Margrit and Carsten Rother and Rhemann, Christoph}
}
@techreport {Singaraju,
	title = {Supplementary material for New Appearance Models for Image Matting},
	year = {2009},
	author = {Singaraju, D and Carsten Rother and Rhemann, C}
}
@article {Shotton2009,
	title = {TextonBoost for image understanding: Multi-class object recognition and segmentation by jointly modeling texture, layout, and context},
	journal = {International Journal of Computer Vision},
	volume = {81},
	number = {1},
	year = {2009},
	pages = {2{\textendash}23},
	abstract = {This paper details a new approach for learning a discriminative model of object classes, incorporating texture, layout, and context information efficiently. The learned model is used for automatic visual understanding and semantic segmentation of photographs. Our discriminative model exploits texture-layout filters, novel features based on textons, which jointly model patterns of texture and their spatial layout. Unary classification and feature selection is achieved using shared boosting to give an efficient classifier which can be applied to a large number of classes. Accurate image segmentation is achieved by incorporating the unary classifier in a conditional random field, which (i) captures the spatial interactions between class labels of neighboring pixels, and (ii) improves the segmentation of specific object instances. Efficient training of the model on large datasets is achieved by exploiting both random feature selection and piecewise training methods. High classification and segmentation accuracy is demonstrated on four varied databases: (i) the MSRC 21-class database containing photographs of real objects viewed under general lighting conditions, poses and viewpoints, (ii) the 7-class Corel subset and (iii) the 7-class Sowerby database used in He et al. (Proceeding of IEEE Conference on Computer Vision and Pattern Recognition, vol. 2, pp. 695-702, June 2004), and (iv) a set of video sequences of television shows. The proposed algorithm gives competitive and visually pleasing results for objects that are highly textured (grass, trees, etc.), highly structured (cars, faces, bicycles, airplanes, etc.), and even articulated (body, cow, etc.). {\textcopyright} 2007 Springer Science+Business Media, LLC.},
	keywords = {Boosting, Conditional random field, Context, image understanding, Layout, Object recognition, Piecewise training, Segmentation, Semantic image segmentation, Textons, Texture},
	issn = {09205691},
	doi = {10.1007/s11263-007-0109-1},
	url = {http://jamie.shotton.org/work/code.html},
	author = {Shotton, Jamie and Winn, John and Carsten Rother and Criminisi, Antonio}
}
@conference {Nguyen2009a,
	title = {Weakly supervised discriminative localization and classification: A joint learning process},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2009},
	pages = {1925{\textendash}1932},
	abstract = {Visual categorization problems, such as object classification or action recognition, are increasingly often approached using a detection strategy: a classifier function is first applied to candidate subwindows of the image or the video, and then the maximum classifier score is used for class decision. Traditionally, the subwindow classifiers are trained on a large collection of examples manually annotated with masks or bounding boxes. The reliance on time-consuming human labeling effectively limits the application of these methods to problems involving very few categories. Furthermore, the human selection of the masks introduces arbitrary biases (e.g. in terms of window size and location) which may be suboptimal for classification. In this paper we propose a novel method for learning a discriminative subwindow classifier from examples annotated with binary labels indicating the presence of an object or action of interest, but not its location. During training, our approach simultaneously localizes the instances of the positive class and learns a subwindow SVM to recognize them. We extend our method to classification of time series by presenting an algorithm that localizes the most discriminative set of temporal segments in the signal. We evaluate our approach on several datasets for object and action recognition and show that it achieves results similar and in many cases superior to those obtained with full supervision. {\textcopyright}2009 IEEE.},
	isbn = {9781424444205},
	doi = {10.1109/ICCV.2009.5459426},
	author = {Nguyen, Minh Hoai and Torresani, Lorenzo and De La Torre, Fernando and Carsten Rother}
}
@conference {Nguyen2009,
	title = {Weakly supervised discriminative localization and classification: A joint learning process},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2009},
	pages = {1925{\textendash}1932},
	abstract = {Visual categorization problems, such as object classification or action recognition, are increasingly often approached using a detection strategy: a classifier function is first applied to candidate subwindows of the image or the video, and then the maximum classifier score is used for class decision. Traditionally, the subwindow classifiers are trained on a large collection of examples manually annotated with masks or bounding boxes. The reliance on time-consuming human labeling effectively limits the application of these methods to problems involving very few categories. Furthermore, the human selection of the masks introduces arbitrary biases (e.g. in terms of window size and location) which may be suboptimal for classification. In this paper we propose a novel method for learning a discriminative subwindow classifier from examples annotated with binary labels indicating the presence of an object or action of interest, but not its location. During training, our approach simultaneously localizes the instances of the positive class and learns a subwindow SVM to recognize them. We extend our method to classification of time series by presenting an algorithm that localizes the most discriminative set of temporal segments in the signal. We evaluate our approach on several datasets for object and action recognition and show that it achieves results similar and in many cases superior to those obtained with full supervision. {\textcopyright}2009 IEEE.},
	isbn = {9781424444205},
	doi = {10.1109/ICCV.2009.5459426},
	author = {Nguyen, Minh Hoai and Torresani, Lorenzo and De La Torre, Fernando and Carsten Rother}
}
@conference {Gehler2008,
	title = {Bayesian color constancy revisited},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {Computational color constancy is the task of estimating the true reflectances of visible surfaces in an image. In this paper we follow a line of research that assumes uniform illumination of a scene, and that the principal step in estimating reflectances is the estimation of the scene illuminant. We review recent approaches to illuminant estimation, firstly those based on formulae for normalisation of the reflectance distribution in an image - so-called grey-world algorithms, and those based on a Bayesian formulation of image formation. In evaluating these previous approaches we introduce a new tool in the form of a database of 568 high-quality, indoor and outdoor images, accurately labelled with illuminant, and preserved in their raw form, free of correction or normalisation. This has enabled us to establish several properties experimentally. Firstly automatic selection of grey-world algorithms according to image properties is not nearly so effective as has been thought. Secondly, it is shown that Bayesian illuminant estimation is significantly improved by the improved accuracy of priors for illuminant and reflectance that are obtained from the new dataset. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587765},
	author = {Gehler, Peter Vincent and Carsten Rother and Blake, Andrew and Minka, Tom and Sharp, Toby}
}
@article {Szeliski2008a,
	title = {A comparative study of energy minimization methods for Markov random fields with smoothness-based priors},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {30},
	number = {6},
	year = {2008},
	month = {jun},
	pages = {1068{\textendash}1080},
	abstract = {Among the most exciting advances in early vision has been the development of efficient energy minimization algorithms for pixel-labeling tasks such as depth or texture computation. It has been known for decades that such problems can be elegantly expressed as Markov random fields, yet the resulting energy minimization problems have been widely viewed as intractable. Recently, algorithms such as graph cuts and loopy belief propagation (LBP) have proven to be very powerful: for example, such methods form the basis for almost all the top-performing stereo methods. However, the tradeoffs among different energy minimization algorithms are still not well understood. In this paper we describe a set of energy minimization benchmarks and use them to compare the solution quality and running time of several common energy minimization algorithms. We investigate three promising recent methods graph cuts, LBP, and tree-reweighted message passing in addition to the well-known older iterated conditional modes (ICM) algorithm. Our benchmark problems are drawn from published energy functions used for stereo, image stitching, interactive segmentation, and denoising. We also provide a general-purpose software interface that allows vision researchers to easily switch between optimization methods. Benchmarks, code, images, and results are available at http://vision.middlebury.edu/ MRF/. {\textcopyright} 2008 IEEE.},
	keywords = {Belief propagation, Global optimization, Graph Cuts, Markov random fields, Performance evaluation},
	issn = {01628828},
	doi = {10.1109/TPAMI.2007.70844},
	author = {Szeliski, Richard and Zabih, Ramin and Scharstein, Daniel and Veksler, Olga and Kolmogorov, Vladimir and Agarwala, Aseem and Tappen, Marshall and Carsten Rother}
}
@article {Szeliski2008,
	title = {A comparative study of energy minimization methods for Markov random fields with smoothness-based priors},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {30},
	number = {6},
	year = {2008},
	pages = {1068{\textendash}1080},
	publisher = {Springer-Verlag},
	abstract = {Among the most exciting advances in early vision has been the development of efficient energy minimization algorithms for pixel-labeling tasks such as depth or texture computation. It has been known for decades that such problems can be elegantly expressed as Markov random fields, yet the resulting energy minimization problems have been widely viewed as intractable. Recently, algorithms such as graph cuts and loopy belief propagation (LBP) have proven to be very powerful: for example, such methods form the basis for almost all the top-performing stereo methods. However, the tradeoffs among different energy minimization algorithms are still not well understood. In this paper we describe a set of energy minimization benchmarks and use them to compare the solution quality and running time of several common energy minimization algorithms. We investigate three promising recent methods graph cuts, LBP, and tree-reweighted message passing in addition to the well-known older iterated conditional modes (ICM) algorithm. Our benchmark problems are drawn from published energy functions used for stereo, image stitching, interactive segmentation, and denoising. We also provide a general-purpose software interface that allows vision researchers to easily switch between optimization methods. Benchmarks, code, images, and results are available at http://vision.middlebury.edu/ MRF/. {\textcopyright} 2008 IEEE.},
	keywords = {Belief propagation, Global optimization, Graph Cuts, Markov random fields, Performance evaluation},
	issn = {01628828},
	doi = {10.1109/TPAMI.2007.70844},
	url = {http://vision.middlebury.edu/MRF.},
	author = {Szeliski, Richard and Zabih, Ramin and Scharstein, Daniel and Veksler, Olga and Kolmogorov, Vladimir and Agarwala, Aseem and Tappen, Marshall and Carsten Rother}
}
@conference {Torresani2008a,
	title = {Feature correspondence via graph matching: Models and global optimization},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {5303 LNCS},
	number = {PART 2},
	year = {2008},
	pages = {596{\textendash}609},
	abstract = {In this paper we present a new approach for establishing correspondences between sparse image features related by an unknown non-rigid mapping and corrupted by clutter and occlusion, such as points extracted from a pair of images containing a human figure in distinct poses. We formulate this matching task as an energy minimization problem by defining a complex objective function of the appearance and the spatial arrangement of the features. Optimization of this energy is an instance of graph matching, which is in general a NP-hard problem. We describe a novel graph matching optimization technique, which we refer to as dual decomposition (DD), and demonstrate on a variety of examples that this method outperforms existing graph matching algorithms. In the majority of our examples DD is able to find the global minimum within a minute. The ability to globally optimize the objective allows us to accurately learn the parameters of our matching model from training examples. We show on several matching tasks that our learned model yields results superior to those of state-of-the-art methods. {\textcopyright} 2008 Springer Berlin Heidelberg.},
	isbn = {3540886850},
	issn = {03029743},
	doi = {10.1007/978-3-540-88688-4-44},
	author = {Torresani, Lorenzo and Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Torresani2008,
	title = {Feature correspondence via graph matching: Models and global optimization},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {5303 LNCS},
	number = {PART 2},
	year = {2008},
	pages = {596{\textendash}609},
	abstract = {In this paper we present a new approach for establishing correspondences between sparse image features related by an unknown non-rigid mapping and corrupted by clutter and occlusion, such as points extracted from a pair of images containing a human figure in distinct poses. We formulate this matching task as an energy minimization problem by defining a complex objective function of the appearance and the spatial arrangement of the features. Optimization of this energy is an instance of graph matching, which is in general a NP-hard problem. We describe a novel graph matching optimization technique, which we refer to as dual decomposition (DD), and demonstrate on a variety of examples that this method outperforms existing graph matching algorithms. In the majority of our examples DD is able to find the global minimum within a minute. The ability to globally optimize the objective allows us to accurately learn the parameters of our matching model from training examples. We show on several matching tasks that our learned model yields results superior to those of state-of-the-art methods. {\textcopyright} 2008 Springer Berlin Heidelberg.},
	isbn = {3540886850},
	issn = {03029743},
	doi = {10.1007/978-3-540-88688-4-44},
	author = {Torresani, Lorenzo and Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Lempitsky2008,
	title = {FusionFlow: Discrete-continuous optimization for optical flow estimation},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {Accurate estimation of optical flow is a challenging task, which often requires addressing difficult energy optimization problems. To solve them, most top-performing methods rely on continuous optimization algorithms. The modeling accuracy of the energy in this case is often traded for its tractability. This is in contrast to the related problem of narrow-baseline stereo matching, where the top-performing methods employ powerful discrete optimization algorithms such as graph cuts and message-passing to optimize highly non-convex energies. In this paper, we demonstrate how similar non-convex energies can be formulated and optimized discretely in the context of optical flow estimation. Starting with a set of candidate solutions that are produced by fast continuous flow estimation algorithms, the proposed method iteratively fuses these candidate solutions by the computation of minimum cuts on graphs. The obtained continuous-valued fusion result is then further improved using local gradient descent. Experimentally, we demonstrate that the proposed energy is an accurate model and that the proposed discrete-continuous optimization scheme not only finds lower energy solutions than traditional discrete or continuous optimization techniques, but also leads to flow estimates that outperform the current state-of-the-art. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587751},
	author = {Lempitsky, Victor and Roth, Stefan and Carsten Rother}
}
@conference {Vicente2008,
	title = {Graph cut based image segmentation with connectivity priors},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {Graph cut is a popular technique for interactive image segmentation. However, it has certain shortcomings. In particular, graph cut has problems with segmenting thin elongated objects due to the "shrinking bias". To overcome this problem, we propose to impose an additional connectivity prior, which is a very natural assumption about objects. We formulate several versions of the connectivity constraint and show that the corresponding optimization problems are all NP-hard. For some of these versions we propose two optimization algorithms: (i) a practical heuristic technique which we call DijkstraGC, and (ii) a slow method based on problem decomposition which provides a lower bound on the problem. We use the second technique to verify that for some practical examples DijkstraGC is able to find the global minimum. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587440},
	author = {Vicente, Sara and Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Rhemann2008c,
	title = {High resolution matting via interactive trimap segmentation},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {We present a new approach to the matting problem which splits the task into two steps: interactive trimap extraction followed by trimap-based alpha matting. By doing so we gain considerably in terms of speed and quality and are able to deal with high resolution images. This paper has three contributions: (i) a new trimap segmentation method using parametric max-flow; (ii) an alpha matting technique for high resolution images with a new gradient preserving prior on alpha; (iii) a database of 27 ground truth alpha mattes of still objects, which is considerably larger than previous databases and also of higher quality. The database is used to train our system and to validate that both our trimap extraction and our matting method improve on state-of-the-art techniques. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587441},
	author = {Rhemann, Christoph and Carsten Rother and Rav-Acha, Alex and Sharp, Toby}
}
@conference {Rhemann2008b,
	title = {High resolution matting via interactive trimap segmentation},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {We present a new approach to the matting problem which splits the task into two steps: interactive trimap extraction followed by trimap-based alpha matting. By doing so we gain considerably in terms of speed and quality and are able to deal with high resolution images. This paper has three contributions: (i) a new trimap segmentation method using parametric max-flow; (ii) an alpha matting technique for high resolution images with a new gradient preserving prior on alpha; (iii) a database of 27 ground truth alpha mattes of still objects, which is considerably larger than previous databases and also of higher quality. The database is used to train our system and to validate that both our trimap extraction and our matting method improve on state-of-the-art techniques. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587441},
	author = {Rhemann, Christoph and Carsten Rother and Rav-Acha, Alex and Sharp, Toby}
}
@conference {Rhemann2008a,
	title = {High resolution matting via interactive trimap segmentation},
	booktitle = {26th IEEE Conference on Computer Vision and Pattern Recognition, CVPR},
	year = {2008},
	abstract = {We present a new approach to the matting problem which splits the task into two steps: interactive trimap extraction followed by trimap-based alpha matting. By doing so we gain considerably in terms of speed and quality and are able to deal with high resolution images. This paper has three contributions: (i) a new trimap segmentation method using parametric max-flow; (ii) an alpha matting technique for high resolution images with a new gradient preserving prior on alpha; (iii) a database of 27 ground truth alpha mattes of still objects, which is considerably larger than previous databases and also of higher quality. The database is used to train our system and to validate that both our trimap extraction and our matting method improve on state-of-the-art techniques. {\textcopyright}2008 IEEE.},
	isbn = {9781424422432},
	doi = {10.1109/CVPR.2008.4587441},
	author = {Rhemann, Christoph and Carsten Rother and Rav-Acha, Alex and Sharp, Toby}
}
@conference {Lempitsky2008b,
	title = {Image segmentation by branch-and-mincut},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {5305 LNCS},
	number = {PART 4},
	year = {2008},
	pages = {15{\textendash}29},
	abstract = {Efficient global optimization techniques such as graph cut exist for energies corresponding to binary image segmentation from low-level cues. However, introducing a high-level prior such as a shape prior or a color-distribution prior into the segmentation process typically results in an energy that is much harder to optimize. The main contribution of the paper is a new global optimization framework for a wide class of such energies. The framework is built upon two powerful techniques: graph cut and branch-and-bound. These techniques are unified through the derivation of lower bounds on the energies. Being computable via graph cut, these bounds are used to prune branches within a branch-and-bound search. We demonstrate that the new framework can compute globally optimal segmentations for a variety of segmentation scenarios in a reasonable time on a modern CPU. These scenarios include unsupervised segmentation of an object undergoing 3D pose change, category-specific shape segmentation, and the segmentation under intensity/color priors defined by Chan-Vese and GrabCut functionals. {\textcopyright} 2008 Springer Berlin Heidelberg.},
	isbn = {3540886923},
	issn = {03029743},
	doi = {10.1007/978-3-540-88693-8-2},
	author = {Lempitsky, Victor and Blake, Andrew and Carsten Rother}
}
@conference {Lempitsky2008a,
	title = {Image segmentation by branch-and-mincut},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {5305 LNCS},
	number = {PART 4},
	year = {2008},
	pages = {15{\textendash}29},
	abstract = {Efficient global optimization techniques such as graph cut exist for energies corresponding to binary image segmentation from low-level cues. However, introducing a high-level prior such as a shape prior or a color-distribution prior into the segmentation process typically results in an energy that is much harder to optimize. The main contribution of the paper is a new global optimization framework for a wide class of such energies. The framework is built upon two powerful techniques: graph cut and branch-and-bound. These techniques are unified through the derivation of lower bounds on the energies. Being computable via graph cut, these bounds are used to prune branches within a branch-and-bound search. We demonstrate that the new framework can compute globally optimal segmentations for a variety of segmentation scenarios in a reasonable time on a modern CPU. These scenarios include unsupervised segmentation of an object undergoing 3D pose change, category-specific shape segmentation, and the segmentation under intensity/color priors defined by Chan-Vese and GrabCut functionals. {\textcopyright} 2008 Springer Berlin Heidelberg.},
	isbn = {3540886923},
	issn = {03029743},
	doi = {10.1007/978-3-540-88693-8-2},
	author = {Lempitsky, Victor and Blake, Andrew and Carsten Rother}
}
@conference {Rhemann2008,
	title = {Improving color modeling for alpha matting},
	booktitle = {BMVC 2008 - Proceedings of the British Machine Vision Conference 2008},
	year = {2008},
	abstract = {This paper addresses the problem of extracting an alpha matte from a single photograph given a user-defined trimap. A crucial part of this task is the color modeling step where for each pixel the optimal alpha value, together with its confidence, is estimated individually. This forms the data term of the objective function. It comprises of three steps: (i) Collecting a candidate set of potential fore- and background colors; (ii) Selecting high confidence samples from the candidate set; (iii) Estimating a sparsity prior to remove blurry artifacts. We introduce novel ideas for each of these steps and show that our approach considerably improves over state-of-the-art techniques by evaluating it on a large database of 54 images with known high-quality ground truth.},
	doi = {10.5244/C.22.115},
	author = {Rhemann, Christoph and Carsten Rother and Gelautz, Margrit}
}
@conference {Kohli2008,
	title = {On partial optimality in multi-label MRFs},
	booktitle = {Proceedings of the 25th International Conference on Machine Learning},
	year = {2008},
	pages = {480{\textendash}487},
	abstract = {We consider the problem of optimizing multilabel MRFs, which is in general NP-hard and ubiquitous in low-level computer vision. One approach for its solution is to formulate it as an integer linear programming and relax the integrality constraints. The approach we consider in this paper is to first convert the multi-label MRF into an equivalent binary-label MRF and then to relax it. The resulting relaxation can be efficiently solved using a maximum flow algorithm. Its solution provides us with a partially optimal labelling of the binary variables. This partial labelling is then easily transferred to the multi-label problem. We study the theoretical properties of the new relaxation and compare it with the standard one. Specifically, we compare tightness, and characterize a subclass of problems where the two relaxations coincide. We propose several combined algorithms based on the technique and demonstrate their performance on challenging computer vision problems. Copyright 2008 by the author(s)/owner(s).},
	isbn = {9781605582054},
	doi = {10.1145/1390156.1390217},
	author = {Kohli, Pushmeet and Shekhovtsov, Alexander and Carsten Rother and Kolmogorov, Vladimir and Torr, Philip}
}
@conference {Hoiem2007,
	title = {3D LayoutCRF for multi-view object class recognition and segmentation},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2007},
	abstract = {We introduce an approach to accurately detect and segment partially occluded objects in various viewpoints and scales. Our main contribution is a novel framework for combining object-level descriptions (such as position, shape, and color) with pixel-level appearance, boundary, and occlusion reasoning. In training, we exploit a rough 3D object model to learn physically localized part appearances. To find and segment objects in an image, we generate proposals based on the appearance and layout of local parts. The proposals are then refined after incorporating object-level information, and overlapping objects compete for pixels to produce a final description and segmentation of objects in the scene. A further contribution is a novel instance penalty, which is handled very efficiently during inference. We experimentally validate our approach on the challenging PASCAL{\textquoteright}06 car database. {\textcopyright} 2007 IEEE.},
	isbn = {1424411807},
	issn = {10636919},
	doi = {10.1109/CVPR.2007.383045},
	author = {Hoiem, Derek and Carsten Rother and Winn, John}
}
@conference {Kolmogorov2007b,
	title = {Applications of parametric maxflow in computer vision},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2007},
	abstract = {The maximum flow algorithm for minimizing energy functions of binary variables has become a standard tool in computer vision. In many cases, unary costs of the energy depend linearly on parameter λ. In this paper we study vision applications for which it is important to solve the max-flow problem for different λ{\textquoteright}s. An example is a weighting between data and regularization terms in image segmentation or stereo: it is desirable to vary it both during training (to learn λ from ground truth data) and testing (to select best λ using high-knowledge constraints, e.g. user input). We review algorithmic aspects of this parametric maximum flow problem previously unknown in vision, such as the ability to compute all breakpoints of λ and corresponding optimal configurations in finite time. These results allow, in particular, to minimize the ratio of some geometric functionals, such as flux of a vector field over length (or area). Previously, such functionals were tackled with shortest path techniques applicable only in 2D. We give theoretical improvements for "PDE cuts" [5]. We present experimental results for image segmentation, 3D reconstruction, and the cosegmentation problem. {\textcopyright}2007 IEEE.},
	doi = {10.1109/ICCV.2007.4408910},
	author = {Kolmogorov, Vladimir and Boykov, Yuri and Carsten Rother}
}
@conference {Kolmogorov2007a,
	title = {Applications of parametric maxflow in computer vision},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2007},
	abstract = {The maximum flow algorithm for minimizing energy functions of binary variables has become a standard tool in computer vision. In many cases, unary costs of the energy depend linearly on parameter λ. In this paper we study vision applications for which it is important to solve the max-flow problem for different λ{\textquoteright}s. An example is a weighting between data and regularization terms in image segmentation or stereo: it is desirable to vary it both during training (to learn λ from ground truth data) and testing (to select best λ using high-knowledge constraints, e.g. user input). We review algorithmic aspects of this parametric maximum flow problem previously unknown in vision, such as the ability to compute all breakpoints of λ and corresponding optimal configurations in finite time. These results allow, in particular, to minimize the ratio of some geometric functionals, such as flux of a vector field over length (or area). Previously, such functionals were tackled with shortest path techniques applicable only in 2D. We give theoretical improvements for "PDE cuts" [5]. We present experimental results for image segmentation, 3D reconstruction, and the cosegmentation problem. {\textcopyright}2007 IEEE.},
	doi = {10.1109/ICCV.2007.4408910},
	author = {Kolmogorov, Vladimir and Boykov, Yuri and Carsten Rother}
}
@conference {Kannan2007a,
	title = {Clustering appearance and shape by learning jigsaws},
	booktitle = {Advances in Neural Information Processing Systems},
	year = {2007},
	pages = {657{\textendash}664},
	abstract = {Patch-based appearance models are used in a wide range of computer vision applications. To learn such models it has previously been necessary to specify a suitable set of patch sizes and shapes by hand. In the jigsaw model presented here, the shape, size and appearance of patches are learned automatically from the repeated structures in a set of training images. By learning such irregularly shaped {\textquoteright}jigsaw pieces{\textquoteright}, we are able to discover both the shape and the appearance of object parts without supervision. When applied to face images, for example, the learned jigsaw pieces are surprisingly strongly associated with face parts of different shapes and scales such as eyes, noses, eyebrows and cheeks, to name a few. We conclude that learning the shape of the patch not only improves the accuracy of appearance-based part detection but also allows for shape-based part detection. This enables parts of similar appearance but different shapes to be distinguished; for example, while foreheads and cheeks are both skin colored, they have markedly different shapes.},
	isbn = {9780262195683},
	issn = {10495258},
	doi = {10.7551/mitpress/7503.003.0087},
	author = {Kannan, Anitha and Winn, John and Carsten Rother}
}
@conference {Kannan2007,
	title = {Clustering appearance and shape by learning jigsaws},
	booktitle = {Advances in Neural Information Processing Systems},
	year = {2007},
	pages = {657{\textendash}664},
	abstract = {Patch-based appearance models are used in a wide range of computer vision applications. To learn such models it has previously been necessary to specify a suitable set of patch sizes and shapes by hand. In the jigsaw model presented here, the shape, size and appearance of patches are learned automatically from the repeated structures in a set of training images. By learning such irregularly shaped {\textquoteright}jigsaw pieces{\textquoteright}, we are able to discover both the shape and the appearance of object parts without supervision. When applied to face images, for example, the learned jigsaw pieces are surprisingly strongly associated with face parts of different shapes and scales such as eyes, noses, eyebrows and cheeks, to name a few. We conclude that learning the shape of the patch not only improves the accuracy of appearance-based part detection but also allows for shape-based part detection. This enables parts of similar appearance but different shapes to be distinguished; for example, while foreheads and cheeks are both skin colored, they have markedly different shapes.},
	isbn = {9780262195683},
	issn = {10495258},
	doi = {10.7551/mitpress/7503.003.0087},
	author = {Kannan, Anitha and Winn, John and Carsten Rother}
}
@conference {Sellen2007,
	title = {Do life-logging technologies support memory for the past? An experimental study using sensecam},
	booktitle = {Conference on Human Factors in Computing Systems - Proceedings},
	year = {2007},
	pages = {81{\textendash}90},
	abstract = {We report on the results of a study using SenseCam, a "life- logging" technology in the form of a wearable camera, which aims to capture data about everyday life in order to support people{\textquoteright}s memory for past, personal events. We find evidence that SenseCam images do facilitate people{\textquoteright}s ability to connect to their past, but that images do this in different ways. We make a distinction between "remembering" the past, and "knowing" about it, and provide evidence that SenseCam images work differently over time in these capacities. We also compare the efficacy of user-captured images with automatically captured images and discuss the implications of these findings and others for how we conceive of and make claims about life-logging technologies. {\textcopyright} Copyright 2007 ACM.},
	keywords = {Capture, Episodic or autobiographical memory, Images, Life-logging, Personal digital archives, SenseCam},
	isbn = {1595935932},
	doi = {10.1145/1240624.1240636},
	author = {Sellen, Abigail and Fogg, Andrew and Aitken, Mike and Hodges, Steve and Carsten Rother and Wood, Ken}
}
@article {Criminisi2007,
	title = {Efficient dense stereo with occlusions for new view-synthesis by four-state dynamic programming},
	journal = {International Journal of Computer Vision},
	volume = {71},
	number = {1},
	year = {2007},
	pages = {89{\textendash}110},
	publisher = {Kluwer Academic Publishers},
	abstract = {A new algorithm is proposed for efficient stereo and novel view synthesis. Given the video streams acquired by two synchronized cameras the proposed algorithm synthesises images from a virtual camera in arbitrary position near the physical cameras. The new technique is based on an improved, dynamic-programming, stereo algorithm for efficient novel view generation. The two main contributions of this paper are: (i) a new four state matching graph for dense stereo dynamic programming, that supports accurate occlusion labelling; (ii) a compact geometric derivation for novel view synthesis by direct projection of the minimum cost surface. Furthermore, the paper presents an algorithm for the temporal maintenance of a background model to enhance the rendering of occlusions and reduce temporal artefacts (flicker); and a cost aggregation algorithm that acts directly in the three-dimensional matching cost space. The proposed algorithm has been designed to work with input images with large disparity range, a common practical situation. The enhanced occlusion handling capabilities of the new dynamic programming algorithm are evaluated against those of the most powerful state-of-the-art dynamic programming and graph-cut techniques. Four-state DP is also evaluated against the disparity-based Middlebury error metrics and its performance found to be amongst the best of the efficient algorithms. A number of examples demonstrate the robustness of four-state DP to artefacts in stereo video streams. This includes demonstrations of cyclopean view synthesis in extended conversational sequences, synthesis from a freely translating virtual camera and, finally, basic 3D scene editing. {\textcopyright} 2006 Springer Science + Business Media, LLC.},
	keywords = {Dense stereo, Gaze correction, Image-based rendering, Video-conferencing},
	issn = {09205691},
	doi = {10.1007/s11263-006-8525-1},
	author = {Criminisi, A and Blake, A and Carsten Rother and Shotton, J and Torr, P. H.S.}
}
@article {Blake2007,
	title = {Fusion of stereo, colour and contrast},
	journal = {Springer Tracts in Advanced Robotics},
	volume = {28},
	year = {2007},
	issn = {16107438},
	doi = {10.1007/978-3-540-48113-3_27},
	url = {www.research.microsoft.com/vision/cambridge},
	author = {Blake, A and Criminisi, A and Cross, G and Kolmogorov, V and Carsten Rother}
}
@conference {Lempitsky2007,
	title = {LogCut - Efficient graph cut optimization for markov random fields},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	year = {2007},
	abstract = {Markov Random Fields (MRFs) are ubiquitous in low-level computer vision. In this paper, we propose a new approach to the optimization of multi-labeled MRFs. Similarly to α-expansion it is based on iterative application of binary graph cut. However, the number of binary graph cuts required to compute a labelling grows only logarithmically with the size of label space, instead of linearly. We demonstrate that for applications such as optical flow, image restoration, and high resolution stereo, this gives an order of magnitude speed-up, for comparable energies. Iterations are performed by "fusion" of solutions, done with QPBO which is related to graph cut but can deal with non-submodularity. At convergence, the method achieves optima on a par with the best competitors, and sometimes even exceeds them. {\textcopyright}2007 IEEE.},
	doi = {10.1109/ICCV.2007.4408907},
	author = {Lempitsky, Victor and Carsten Rother and Blake, Andrew}
}
@booklet {Kolmogorov2007,
	title = {Minimizing nonsubmodular functions with graph cuts - A review},
	howpublished = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {29},
	number = {7},
	year = {2007},
	pages = {1274{\textendash}1279},
	abstract = {Optimization techniques based on graph cuts have become a standard tool for many vision applications. These techniques allow to minimize efficiently certain energy functions corresponding to pairwise Markov Random Fields (MRFs). Currently, there is an accepted view within the computer vision community that graph cuts can only be used for optimizing a limited class of MRF energies (e.g., submodular functions). In this survey, we review some results that show that graph cuts can be applied to a much larger class of energy functions (in particular, nonsubmodular functions). While these results are well-known in the optimization community, to our knowledge they were not used in the context of computer vision and MRF optimization. We demonstrate the relevance of these results to vision on the problem of binary texture restoration. {\textcopyright} 2007 IEEE.},
	keywords = {energy minimization, Markov random fields, Min cut/max flow, Quadratic pseudo-Boolean optimization, Texture restoration},
	issn = {01628828},
	doi = {10.1109/TPAMI.2007.1031},
	author = {Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Rother2007a,
	title = {Optimizing binary MRFs via extended roof duality},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	year = {2007},
	abstract = {Many computer vision applications rely on the efficient optimization of challenging, so-called non-submodular, binary pairwise MRFs. A promising graph cut based approach for optimizing such MRFs known as "roof duality" was recently introduced into computer vision. We study two methods which extend this approach. First, we discuss an efficient implementation of the "probing" technique introduced recently by Boros et al. [5]. It simplifies the MRF while preserving the global optimum. Our code is 400-700 faster on some graphs than the implementation of [5]. Second, we present a new technique which takes an arbitrary input labeling and tries to improve its energy. We give theoretical characterizations of local minima of this procedure. We applied both techniques to many applications, including image segmentation, new view synthesis, superresolution, diagram recognition, parameter learning, texture restoration, and image deconvolution. For several applications we see that we are able to find the global minimum very efficiently, and considerably outperform the original roof duality approach. In comparison to existing techniques, such as graph cut, TRW, BP, ICM, and simulated annealing, we nearly always find a lower energy. {\textcopyright} 2007 IEEE.},
	isbn = {1424411807},
	issn = {10636919},
	doi = {10.1109/CVPR.2007.383203},
	author = {Carsten Rother and Kolmogorov, Vladimir and Lempitsky, Victor and Szummer, Martin}
}
@article {Rother2007,
	title = {Optimizing Binary MRFs via Extended Roof Duality Technical Report MSR-TR-2007-46},
	journal = {Computing},
	year = {2007},
	abstract = {Many computer vision applications rely on the efficient optimization of challenging, so-called non-submodular, binary pairwise MRFs. A promising graph cut based approach for optimizing such MRFs known as "roof duality" was recently introduced into computer vision. We study two methods which extend this approach. First, we discuss an efficient implementation of the "probing" technique introduced recently by Boros et al. [8]. It simplifies the MRF while preserving the global optimum. Our code is 400-700 faster on some graphs than the implementation of [8]. Second , we present a new technique which takes an arbitrary input labeling and tries to improve its energy. We give theoretical characterizations of local minima of this procedure. We applied both techniques to many applications, including image segmentation, new view synthesis, super-resolution, diagram recognition, parameter learning, texture restoration, and image deconvolution. For several applications we see that we are able to find the global minimum very efficiently, and considerably outperform the original roof duality approach. In comparison to existing techniques , such as graph cut, TRW, BP, ICM, and simulated annealing, we nearly always find a lower energy.},
	url = {http://research.microsoft.com/vision/cambridge/},
	author = {Carsten Rother and Kolmogorov, Vladimir and Lempitsky, Victor and Szummer, Martin}
}
@conference {Lalonde2007,
	title = {Photo clip art},
	booktitle = {Proceedings of the ACM SIGGRAPH Conference on Computer Graphics},
	year = {2007},
	abstract = {We present a system for inserting new objects into existing photographs by querying a vast image-based object library, pre-computed using a publicly available Internet object database. The central goal is to shield the user from all of the arduous tasks typically involved in image compositing. The user is only asked to do two simple things: 1) pick a 3D location in the scene to place a new object; 2) select an object to insert using a hierarchical menu. We pose the problem of object insertion as a data-driven, 3D-based, context-sensitive object retrieval task. Instead of trying to manipulate the object to change its orientation, color distribution, etc. to fit the new image, we simply retrieve an object of a specified class that has all the required properties (camera pose, lighting, resolution, etc) from our large object library. We present new automatic algorithms for improving object segmentation and blending, estimating true 3D object size and orientation, and estimating scene lighting conditions. We also present an intuitive user interface that makes object insertion fast and simple even for the artistically challenged. {\textcopyright} 2007 ACM.},
	keywords = {3D scene reasoning, Blending and compositing, Computational photography, Image databases, Object insertion},
	doi = {10.1145/1275808.1276381},
	url = {http://graphics.cs.cmu.edu/projects/photoclipart/},
	author = {Lalonde, Jean Fran{\c c}ois and Hoiem, Derek and Efros, Alexei A and Carsten Rother and Winn, John and Criminisi, Antonio}
}
@conference {Kolmogorov2006b,
	title = {Comparison of energy minimization algorithms for highly connected graphs},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {3952 LNCS},
	year = {2006},
	pages = {1{\textendash}15},
	abstract = {Algorithms for discrete energy minimization play a fundamental role for low-level vision. Known techniques include graph cuts, belief propagation (BP) and recently introduced tree-reweighted message passing (TRW). So far, the standard benchmark for their comparison has been a 4-connected grid-graph arising in pixel-labelling stereo. This minimization problem, however, has been largely solved: recent work shows that for many scenes TRW finds the global optimum. Furthermore, it is known that a 4-connecled grid-graph is a poor stereo model since it does not take occlusions into account. We propose the problem of stereo with occlusions as a new test bed for minimization algorithms. This is a more challenging graph since it has much larger connectivity, and it also serves as a better stereo model. An attractive feature of this problem is that increased connectivity does not result in increased complexity of message passing algorithms. Indeed, one contribution of this paper is to show that sophisticated implementations of BP and TRW have the same time and memory complexity as that of 4-connecled grid-graph stereo. The main conclusion of our experimental study is that for our problem graph cut outperforms both TRW and BP considerably. TRW achieves consistently a lower energy than BP. However, as connectivity increases the speed of convergence of TRW becomes slower. Unlike 4-connected grids, the difference between the energy of the best optimization method and the lower bound of TRW appears significant. This shows the hardness of the problem and motivates future research. {\textcopyright} Springer-Verlag Berlin Heidelberg 2006.},
	isbn = {3540338349},
	issn = {16113349},
	doi = {10.1007/11744047_1},
	author = {Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Kolmogorov2006a,
	title = {Comparison of energy minimization algorithms for highly connected graphs},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {3952 LNCS},
	year = {2006},
	pages = {1{\textendash}15},
	abstract = {Algorithms for discrete energy minimization play a fundamental role for low-level vision. Known techniques include graph cuts, belief propagation (BP) and recently introduced tree-reweighted message passing (TRW). So far, the standard benchmark for their comparison has been a 4-connected grid-graph arising in pixel-labelling stereo. This minimization problem, however, has been largely solved: recent work shows that for many scenes TRW finds the global optimum. Furthermore, it is known that a 4-connecled grid-graph is a poor stereo model since it does not take occlusions into account. We propose the problem of stereo with occlusions as a new test bed for minimization algorithms. This is a more challenging graph since it has much larger connectivity, and it also serves as a better stereo model. An attractive feature of this problem is that increased connectivity does not result in increased complexity of message passing algorithms. Indeed, one contribution of this paper is to show that sophisticated implementations of BP and TRW have the same time and memory complexity as that of 4-connecled grid-graph stereo. The main conclusion of our experimental study is that for our problem graph cut outperforms both TRW and BP considerably. TRW achieves consistently a lower energy than BP. However, as connectivity increases the speed of convergence of TRW becomes slower. Unlike 4-connected grids, the difference between the energy of the best optimization method and the lower bound of TRW appears significant. This shows the hardness of the problem and motivates future research. {\textcopyright} Springer-Verlag Berlin Heidelberg 2006.},
	isbn = {3540338349},
	issn = {16113349},
	doi = {10.1007/11744047_1},
	author = {Kolmogorov, Vladimir and Carsten Rother}
}
@conference {Rother2006,
	title = {Cosegmentation of image pairs by histogram matching - Incorporating a global constraint into MRFs},
	booktitle = {Proceedings of the IEEE Computer Society Conference on Computer Vision and Pattern Recognition},
	volume = {1},
	year = {2006},
	pages = {994{\textendash}1000},
	abstract = {We introduce the term cosegmentation which denotes the task of segmenting simultaneously the common parts of an image pair. A generative model for cosegmentation is presented. Inference in the model leads to minimizing an energy with an MRF term encoding spatial coherency and a global constraint which attempts to match the appearance histograms of the common parts. This energy has not been proposed previously and its optimization is challenging and NP-hard. For this problem a novel optimization scheme which we call trust region graph cuts is presented. We demonstrate that this framework has the potential to improve a wide range of research: Object driven image retrieval, video tracking and segmentation, and interactive image editing. The power of the framework lies in its generality, the common part can be a rigid/non-rigid object (or scene), observed from different viewpoints or even similar objects of the same class. {\textcopyright} 2006 IEEE.},
	isbn = {0769525970},
	issn = {10636919},
	doi = {10.1109/CVPR.2006.91},
	url = {http://research.microsoft.com/vision/cambridge/},
	author = {Carsten Rother and Kolmogorov, Vladimir and Minka, Tom and Blake, Andrew}
}
@article {Kolmogorov2006,
	title = {Probabilistic fusion of stereo with color and contrast for bilayer segmentation},
	journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
	volume = {28},
	number = {9},
	year = {2006},
	pages = {1480{\textendash}1492},
	abstract = {This paper describes models and algorithms for the real-time segmentation of foreground from background layers in stereo video sequences. Automatic separation of layers from color/contrast or from stereo alone is known to be error-prone. Here, color, contrast, and stereo matching information are fused to infer layers accurately and efficiently. The first algorithm, Layered Dynamic Programming (LDP), solves stereo in an extended six-state space that represents both foreground/background layers and occluded regions. The stereo-match likelihood is then fused with a contrast-sensitive color model that is learned on-the-fly and stereo disparities are obtained by dynamic programming. The second algorithm, Layered Graph Cut (LGC), does not directly solve stereo. Instead, the stereo match likelihood is marginalized over disparities to evaluate foreground and background hypotheses and then fused with a contrast-sensitive color model like the one used in LDP. Segmentation is solved efficiently by ternary graph cut. Both algorithms are evaluated with respect to ground truth data and found to have similar performance, substantially better than either stereo or color/contrast alone. However, their characteristics with respect to computational efficiency are rather different. The algorithms are demonstrated in the application of background substitution and shown to give good quality composite video output. {\textcopyright} 2006 IEEE.},
	keywords = {3D/stereo scene analysis, Computer vision, Dynamic programming, Image processing and computer vision, Parameter learning},
	issn = {01628828},
	doi = {10.1109/TPAMI.2006.193},
	url = {http://research.microsoft.com/vision/cambridge},
	author = {Kolmogorov, Vladimir and Criminisi, Antonio and Blake, Andrew and Cross, Geoffrey and Carsten Rother}
}
@conference {Shotton2006,
	title = {TextonBoost: Joint appearance, shape and context modeling for multi-class object recognition and segmentation},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {3951 LNCS},
	year = {2006},
	pages = {1{\textendash}15},
	abstract = {This paper proposes a new approach to learning a discriminative model of object classes, incorporating appearance, shape and context information efficiently. The learned model is used for automatic visual recognition and semantic segmentation of photographs. Our discriminative model exploits novel features, based on textons, which jointly model shape and texture. Unary classification and feature selection is achieved using shared boosting to give an efficient classifier which can be applied to a large number of classes. Accurate image segmentation is achieved by incorporating these classifiers in a conditional random field. Efficient training of the model on very large datasets is achieved by exploiting both random feature selection and piecewise training methods. High classification and segmentation accuracy are demonstrated on three different databases: i) our own 21-object class database of photographs of real objects viewed under general lighting conditions, poses and viewpoints, ii) the 7-class Corel subset and iii) the 7-class Sowerby database used in [1]. The proposed algorithm gives competitive results both for highly textured (e.g. grass, trees), highly structured (e.g. cars, faces, bikes, aeroplanes) and articulated objects (e.g. body, cow). {\textcopyright} Springer-Verlag Berlin Heidelberg 2006.},
	keywords = {aeroplanes) and articulated objects (eg body, bikes, cow), faces, g grass, highly structured (eg cars, trees)},
	isbn = {3540338322},
	issn = {16113349},
	doi = {10.1007/11744023_1},
	url = {http://research.microsoft.com/vision/cambridge/recognition/.},
	author = {Shotton, Jamie and Winn, John and Carsten Rother and Criminisi, Antonio}
}
@conference {Kirk2006,
	title = {Understanding photowork},
	booktitle = {Conference on Human Factors in Computing Systems - Proceedings},
	volume = {2},
	year = {2006},
	pages = {761{\textendash}770},
	abstract = {In this paper we introduce the notion of "photowork" as the activities people perform with their digital photos after capture but prior to end use such as sharing. Surprisingly, these processes of reviewing, downloading, organizing, editing, sorting and filing have received little attention in the literature yet they form the context for a large amount of the {\textquoteright}search{\textquoteright} and {\textquoteright}browse{\textquoteright} activities so commonly referred to in studies of digital photo software. Through a deeper understanding of photowork using field observation and interviews, we seek to highlight its significance as an interaction practice. At the same time, we discover how "search" as it is usually defined may have much less relevance than new ways of browsing for the design of new digital photo tools, in particular, browsing in support of the photowork activities we describe. Copyright 2006 ACM.},
	keywords = {Browsing, Content-based image retrieval, Digital photo albums, Photowork, Searching, Use of images},
	isbn = {1595931783},
	doi = {10.1145/1124772.1124885},
	author = {Kirk, David S and Sellen, Abigail J and Carsten Rother and Wood, Kenneth R}
}
@conference {Kolmogorov2005,
	title = {Bi-layer segmentation of binocular stereo video},
	booktitle = {Proceedings - 2005 IEEE Computer Society Conference on Computer Vision and Pattern Recognition, CVPR 2005},
	volume = {II},
	year = {2005},
	pages = {407{\textendash}414},
	abstract = {This paper describes two algorithms capable of real-time segmentation of foreground from background layers in stereo video sequences. Automatic separation of layers from colour/contrast or from stereo alone is known to be error-prone. Here, colour, contrast and stereo matching information are fused to infer layers accurately and efficiently. The first algorithm, Layered Dynamic Programming (LDP), solves stereo in an extended 6-state space that represents both foreground/background layers and occluded regions. The stereo-match likelihood is then fused with a contrast-sensitive colour model that is learned on the fly, and stereo disparities are obtained by dynamic programming. The second algorithm, Layered Graph Cut (LGC), does not directly solve stereo. Instead the stereo match likelihood is marginalised over foreground and background hypotheses, and fused with a contrast-sensitive colour model like the one used in LDP. Segmentation is solved efficiently by ternary graph cut. Both algorithms are evaluated with respect to ground truth data and found to have similar perfomance, substantially better than stereo or colour/contrast alone. However, their characteristics with respect to computational efficiency are rather different. The algorithms are demonstrated in the application of background substitution and shown to give good quality composite video output.},
	isbn = {0769523722},
	doi = {10.1109/CVPR.2005.91},
	url = {http://research.microsoft.com/vision/cambridge},
	author = {Kolmogorov, V and Criminisi, A and Blake, A and Cross, G and Carsten Rother}
}
@conference {Rother2005,
	title = {Digital tapestry [automatic image synthesis]},
	booktitle = {Computer Vision and Pattern Recognition, 2005. CVPR 2005. IEEE Computer Society Conference on},
	volume = {1},
	year = {2005},
	pages = {589{\textendash}596},
	abstract = {This paper addresses the novel problem of automatically synthesizing an output image from a large collection of different input images. The synthesized image, called a digital tapestry, can be viewed as a visual summary or a virtual {\textquoteright}thumbnail{\textquoteright} of all the images in the input collection. The problem of creating the tapestry is cast as a multi-class labeling problem such that each region in the tapestry is constructed from input image blocks that are salient and such that neighboring blocks satisfy spatial compatibility. This is formulated using a Markov random field and optimized via the graph cut based expansion move algorithm. The standard expansion move algorithm can only handle energies with metric terms, while our energy contains non-metric (soft and hard) constraints. Therefore we propose two novel contributions. First, we extend the expansion move algorithm for energy functions with non-metric hard constraints. Secondly, we modify it for functions with "almost" metric soft terms, and show that it gives good results in practice. The proposed framework was tested on several consumer photograph collections, and the results are presented.},
	isbn = {0-7695-2372-2},
	doi = {10.1109/CVPR.2005.130},
	url = {http://research.microsoft.com/ http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=1467321\%5Cnhttp://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=1467321},
	author = {Carsten Rother and Kumar, Sanjiv and Kolmogorov, Vladimir and Blake, Andrew}
}
@conference {Rother2004,
	title = {"GrabCut" - Interactive foreground extraction using iterated graph cuts},
	booktitle = {ACM Transactions on Graphics},
	volume = {23},
	number = {3},
	year = {2004},
	pages = {309{\textendash}314},
	abstract = {The problem of efficient, interactive foreground/background segmentation in still images is of great practical importance in image editing. Classical image segmentation tools use either texture (colour) information, e.g. Magic Wand, or edge (contrast) information, e.g. Intelligent Scissors. Recently, an approach based on optimization by graph-cut has been developed which successfully combines both types of information. In this paper we extend the graph-cut approach in three respects. First, we have developed a more powerful, iterative version of the optimisation. Secondly, the power of the iterative algorithm is used to simplify substantially the user interaction needed for a given quality of result. Thirdly, a robust algorithm for "border matting" has been developed to estimate simultaneously the alpha-matte around an object boundary and the colours of foreground pixels. We show that for moderately difficult examples the proposed method outperforms competitive tools.},
	keywords = {Alpha Matting, Foreground extraction, Graph Cuts, Image Editing, Interactive Image Segmentation},
	issn = {07300301},
	doi = {10.1145/1015706.1015720},
	author = {Carsten Rother and Kolmogorov, Vladimir and Blake, Andrew}
}
@article {Rother2003a,
	title = {Linear Multi-View Reconstruction for Translating Cameras},
	journal = {Nada.Kth.Se},
	year = {2003},
	abstract = {This paper presents a linear multi view reconstruction algorithm for translating cameras with fixed internal parameters. The main advantages of this method are a) points and camera centers are computed simultaneously from one linear system containing all image data b) the allowance of arbitrary missing data. We show that the key to linearize the SFM problem is the infinite homography which comprises of the cameras{\textquoteright} calibration and rotation. This insight unifies reconstruction methods for calibrated cameras, e.g. Olien-sis [9], and uncalibrated cameras, e.g. Rother-Carlsson [10]. A further contribution of this paper is the summary and comparison of different approaches to determine the infinite homography.},
	url = {http://www.nada.kth.se/ carstenr/papers/paper_ssab03.pdf},
	author = {Carsten Rother}
}
@conference {Rother2003b,
	title = {Linear multi-view reconstruction of points, lines, planes and cameras using a reference plane},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2},
	year = {2003},
	pages = {1210{\textendash}1217},
	abstract = {This paper presents a new linear method for reconstructing simultaneously 3D features (points, lines and planes) and cameras from many perspective views by solving a single linear system. It assumes that a real or virtual reference plane is visible in all views. We call it the Direct Reference Plane (DRP) method. It is well known that the projection relationship between uncalibrated cameras and 3D features is non-linear in the absence of a reference plane. With a known reference plane, points and cameras have a linear relationship, as shown in [16]. The main contribution of this paper is that lines and cameras, as well as, planes and cameras also have a linear relationship. Consequently, all 3D features and all cameras can be reconstructed simultaneously from a single linear system, which handles missing image measurements naturally. A further contribution is an extensive experimental comparison, using real data, of different reference plane and non-reference plane reconstruction methods. For difficult reference plane scenarios, with point or line features, the DRP method is superior to all compared methods. Finally, an extensive list of reference plane scenarios is presented, which shows the wide applicability of the DRP method.},
	doi = {10.1109/iccv.2003.1238629},
	url = {http://www.nada.kth.se/carstenr},
	author = {Carsten Rother}
}
@phdthesis {Rother2003,
	title = {Multi-View Reconstruction and Camera Recovery using a Real or Virtual Reference Plane},
	year = {2003},
	type = {phd},
	abstract = {Reconstructing a 3-dimensional scene from a set of 2-dimensional images is a fundamental problem in computer vision. A systemcapable of performing this task can be used in many applications in robotics, architecture, archaeology, biometrics, human computer interaction and the movie and entertainment industry. Most existing reconstruction approaches exploit one source of information to tackle the problem. This is the motion of the camera, the 2D images are taken from different viewpoints.We exploit an additional information source, the reference plane,whichmakes it possible to reconstruct difficult scenes where other methods fail. A real scene plane may serve as the reference plane. Furthermore, there are many alternative techniques to obtain virtual reference planes. For instance, orthogonal directions in the scene provide a virtual reference plane, the plane at infinity, or images taken with a parallel projection camera. A collection of known and novel reference plane scenarios is presented in this thesis. The main contribution of the thesis is a novel multi-view reconstruction approach us- ing a reference plane. The technique is applicable to three different feature types, points, lines and planes. The novelty of our approach is that all cameras and all features (off the reference plane) are reconstructed simultaneously from a single linear system of im- age measurements. It is based on the novel observation that cameras and features have a linear relationship if a reference plane is known. In the absence of a reference plane, this relationship is non-linear. Thus many previous methods must reconstruct features and cameras sequentially. Another class of methods, popular in the literature, is factorization, but, in contrast to our approach, this has the serious practical drawback that all features are required to be visible in all views. Extensive experiments show that our approach is superior to all previously suggested reference plane and non-reference plane methods for difficult reference plane scenarios. Furthermore, the thesis studies scenes which do not have a unique reconstruction, so- called critical configurations. It is proven that in the presence of a reference plane the set of critical configurations is small. Finally, the thesis introduces a complete, automatic multi-view reconstruction system based on the reference plane approach. The input data is a set of images and the output a 3D point reconstruction together with the corresponding cameras.},
	isbn = {9172834226},
	url = {http://www.google.com/url?sa=t\&rct=j\&q=\&esrc=s\&source=web\&cd=4\&cad=rja\&uact=8\&ved=0CDUQFjAD\&url=http\%3A\%2F\%2Fwww.nada.kth.se\%2Futbildning\%2Fforsk.utb\%2Favhandlingar\%2Fdokt\%2Frother.pdf\&ei=AyX_VPKmIomeNqeOgpgL\&usg=AFQjCNHCmc75P5EHYWLtBUaHtUAs4yOnJw\&bvm=bv.},
	author = {Carsten Rother}
}
@article {Rother2002a,
	title = {Linear multi view reconstruction and camera recovery using a reference plane},
	journal = {International Journal of Computer Vision},
	volume = {49},
	number = {2-3},
	year = {2002},
	pages = {117{\textendash}141},
	abstract = {This paper presents a linear algorithm for simultaneous computation of 3D points and camera positions from multiple perspective views based on having a reference plane visible in all views. The reconstruction and camera recovery is achieved in a single step by finding the null-space of a matrix built from image data using Singular Value Decomposition. Contrary to factorization algorithms this approach does not need to have all points visible in all views. This paper investigates two reference plane configurations: Finite reference planes defined by four coplanar points and infinite reference planes defined by vanishing points. A further contribution of this paper is the study of critical configurations for configurations with four coplanar points. By simultaneously reconstructing points and views we can exploit the numerical stabilizing effect of having wide spread cameras with large mutual baselines. This is demonstrated by reconstructing the outside and inside (courtyard) of a building on the basis of 35 views in one single Singular Value Decomposition.},
	keywords = {Critical configurations, Duality, Missing data, Multiple views, Planar parallax, Projective reconstruction, Reference plane, Structure from motion},
	issn = {09205691},
	doi = {10.1023/A:1020189404787},
	author = {Carsten Rother and Carlsson, Stefan}
}
@conference {Rother2002b,
	title = {Linear multi view reconstruction with missing data},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {2351},
	year = {2002},
	pages = {209{\textendash}324},
	abstract = {General multi view reconstruction from affine or projective cameras has so far been solved most efficiently using methods of factorizing image data matrices into camera and scene parameters. This can be done directly for affine cameras [18] and after computing epipolar geometry for projective cameras [17]. A notorious problem has been the fact that these factorization methods require all points to be visible in all views. This paper presents alternative algorithms for general affine and projective views of multiple points where a) points and camera centers are computed as the nullspace of one linear system constructed from all the image data b) only three points have to be visible in all views. The latter requirement increases the flexibility and usefulness of 3D reconstruction from multiple views. In the case of projective views and unknown epipolar geometry, an additional algorithm is presented which initially assumes affine views and compensates iteratively for the perspective effects. In this paper affine cameras are represented in a projective framework which is novel and leads to a unified treatment of parallel and perspective projection in a single framework. The experiments cover a wide range of different camera motions and compare the presented algorithms to factorization methods, including approaches which handle missing data.},
	keywords = {Affine and Projective Cameras, Linear Multiple View Reconstruction, Missing data, Structure from motion},
	isbn = {9783540437444},
	issn = {16113349},
	doi = {10.1007/3-540-47967-8_21},
	author = {Carsten Rother and Carlsson, Stefan}
}
@conference {Rother2002,
	title = {A new approach to vanishing point detection in architectural environments},
	booktitle = {Image and Vision Computing},
	volume = {20},
	number = {9-10},
	year = {2002},
	pages = {647{\textendash}655},
	abstract = {A man-made environment is characterized by many parallel lines and orthogonal edges. In this article, a new method for detecting the three mutually orthogonal directions of such an environment is presented. Since real-time performance is not necessary for architectural applications, such as building reconstruction, a computationally intensive approach was chosen. However, this enables us to avoid one fundamental error of most other existing techniques. Compared to theirs, our approach is furthermore more rigorous, since all conditions given by three mutually orthogonal directions are identified and utilized. We assume a partly calibrated camera with unknown focal length and unknown principal point. By examining these camera parameters, which can be determined from orthogonal directions, falsely detected vanishing points may be rejected. {\textcopyright} 2002 Elsevier Science B.V. All rights reserved.},
	keywords = {Architecture, Camera calibration, Geometric constraints, Vanishing lines, Vanishing points},
	issn = {02628856},
	doi = {10.1016/S0262-8856(02)00054-9},
	author = {Carsten Rother}
}
@conference {Rother2002c,
	title = {Projective factorization of planes and cameras in multiple views},
	booktitle = {Proceedings - International Conference on Pattern Recognition},
	volume = {16},
	number = {2},
	year = {2002},
	pages = {737{\textendash}740},
	abstract = {This paper proposes a novel method for the projective reconstruction of planes and cameras from multiple images by factorizing a matrix containing all planar homographies between a reference view and all other views. If some planes are not visible in all views an alternative method is presented which solves the problem in two steps: a) all camera centers are recovered simultaneously b) all planes are reconstructed. The key idea of both methods is to specify one of the planes, which is visible in all views, as the plane at infinity. The methods were applied to synthetic and real data, where VRML models can be created with a small amount of user interaction.},
	issn = {10514651},
	doi = {10.1109/icpr.2002.1048408},
	author = {Carsten Rother and Carlsson, Stefan and Tell, Dennis}
}
@conference {Rother2001,
	title = {Linear multi view reconstruction and camera recovery},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {1},
	number = {3},
	year = {2001},
	pages = {42{\textendash}49},
	abstract = {This paper presents a linear algorithm for the simultaneous computation of 3D points and camera positions from multiple perspective views, based on having four points on a reference plane visible in all views. The reconstruction and camera recovery is achieved, in a single step, by finding the null-space of a matrix using singular value decomposition. Unlike factorization algorithms, the presented algorithm does not require all points to be visible in all views. By simultaneously reconstructing points and views the numerically stabilizing effect of having wide spread cameras with large mutual baselines is exploited. Experimental results are presented for both finite and infinite reference planes. An especially interesting application of this method is the reconstruction of architectural scenes with the reference plane taken as the plane at infinity which is visible via three orthogonal vanishing points. This is demonstrated by reconstructing the outside and inside (courtyard) of a building on the basis of 35 views in one single SVD.},
	keywords = {Critical configurations, Duality, Missing data, Multiple views, Planar parallax, Projective reconstruction, Reference plane, Structure from motion},
	doi = {10.1109/iccv.2001.937497},
	author = {Carsten Rother and Carlsson, Stefan}
}
