@conference {AbuAlhaija2019,
	title = {Geometric Image Synthesis},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {11366 LNCS},
	year = {2019},
	pages = {85{\textendash}100},
	abstract = {The task of generating natural images from 3D scenes has been a long standing goal in computer graphics. On the other hand, recent developments in deep neural networks allow for trainable models that can produce natural-looking images with little or no knowledge about the scene structure. While the generated images often consist of realistic looking local patterns, the overall structure of the generated images is often inconsistent. In this work we propose a trainable, geometry-aware image generation method that leverages various types of scene information, including geometry and segmentation, to create realistic looking natural images that match the desired scene structure. Our geometrically-consistent image synthesis method is a deep neural network, called Geometry to Image Synthesis (GIS) framework, which retains the advantages of a trainable method, e.g., differentiability and adaptiveness, but, at the same time, makes a step towards the generalizability, control and quality output of modern graphics rendering engines. We utilize the GIS framework to insert vehicles in outdoor driving scenes, as well as to generate novel views of objects from the Linemod dataset. We qualitatively show that our network is able to generalize beyond the training set to novel scene geometries, object shapes and segmentations. Furthermore, we quantitatively show that the GIS framework can be used to synthesize large amounts of training data which proves beneficial for training instance segmentation models.},
	isbn = {9783030208752},
	issn = {16113349},
	doi = {10.1007/978-3-030-20876-9_6},
	url = {https://youtu.be/W2tFCz9xJoU},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Geiger, Andreas and Carsten Rother}
}
@article {Alhaija2018,
	title = {Augmented Reality Meets Computer Vision},
	journal = {International Journal of Computer Vision},
	volume = {In press},
	year = {2018},
	pages = {1{\textendash}13},
	abstract = {The success of deep learning in computer vision is based on availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment real images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance and a large number of complex object arrangements. In contrast to modeling complete 3D environments, our augmentation approach requires only a few user interactions in combination with 3D shapes of the target object. Through extensive experimentation, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of our approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenes. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on Cityscapes dataset. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amount of annotated real data.},
	keywords = {autonomous driving, data augmenta-, instance segmentation, synthetic training data, tion},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@article {6268,
	title = {Augmented Reality Meets Computer Vision Efficient Data Generation for Urban Driving Scenes},
	journal = {IJCV},
	year = {2018},
	pages = {1-12},
	doi = {10.1007/s11263-018-1070-x},
	author = {Hassan Abu Alhaija and Mustikovela, S K and Mescheder, A and Geiger, C and Carsten Rother}
}
@article {AbuAlhaija2018,
	title = {Augmented Reality Meets Computer Vision: Efficient Data Generation for Urban Driving Scenes},
	journal = {International Journal of Computer Vision},
	volume = {126},
	number = {9},
	year = {2018},
	month = {aug},
	pages = {961{\textendash}972},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation and object detection models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D models of the target object category. Leveraging our approach, we introduce a novel dataset of augmented urban driving scenes with 360 degree images that are used as environment maps to create realistic lighting and reflections on rendered objects. We analyze the significance of realistic object placement by comparing manual placement by humans to automatic methods based on semantic scene analysis. This allows us to create composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. Through an extensive set of experiments, we conclude the right set of parameters to produce augmented data which can maximally enhance the performance of instance segmentation models. Further, we demonstrate the utility of the proposed approach on training standard deep models for semantic instance segmentation and object detection of cars in outdoor driving scenarios. We test the models trained on our augmented data on the KITTI 2015 dataset, which we have annotated with pixel-accurate ground truth, and on the Cityscapes dataset. Our experiments demonstrate that the models trained on augmented imagery generalize better than those trained on fully synthetic data or models trained on limited amounts of annotated real data.},
	keywords = {autonomous driving, Data augmentation, instance segmentation, Object detection, synthetic training data},
	issn = {15731405},
	doi = {10.1007/s11263-018-1070-x},
	url = {http://arxiv.org/abs/1708.01566},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@proceedings {6309,
	title = {Geometric Image Synthesis},
	year = {2018},
	author = {Hassan Abu Alhaija and Mustikovela, S K and Geiger, A and Carsten Rother}
}
@conference {Alhaija2017,
	title = {Augmented reality meets deep learning for car instance segmentation in urban scenes},
	booktitle = {British Machine Vision Conference 2017, BMVC 2017},
	year = {2017},
	abstract = {The success of deep learning in computer vision is based on the availability of large annotated datasets. To lower the need for hand labeled images, virtually rendered 3D worlds have recently gained popularity. Unfortunately, creating realistic 3D content is challenging on its own and requires significant human effort. In this work, we propose an alternative paradigm which combines real and synthetic data for learning semantic instance segmentation models. Exploiting the fact that not all aspects of the scene are equally important for this task, we propose to augment real-world imagery with virtual objects of the target category. Capturing real-world images at large scale is easy and cheap, and directly provides real background appearances without the need for creating complex 3D models of the environment. We present an efficient procedure to augment these images with virtual objects. This allows us to create realistic composite images which exhibit both realistic background appearance as well as a large number of complex object arrangements. In contrast to modeling complete 3D environments, our data augmentation approach requires only a few user interactions in combination with 3D shapes of the target object category. We demonstrate the utility of the proposed approach for training a state-of-the-art high-capacity deep model for semantic instance segmentation. In particular, we consider the task of segmenting car instances on the KITTI dataset which we have annotated with pixel-accurate ground truth. Our experiments demonstrate that models trained on augmented imagery generalize better than those trained on synthetic data or models trained on limited amounts of annotated real data.},
	isbn = {190172560X},
	author = {Hassan Abu Alhaija and Mustikovela, Siva Karthik and Mescheder, Lars and Geiger, Andreas and Carsten Rother}
}
@conference {Behl2017a,
	title = {Bounding Boxes, Segmentations and Object Coordinates: How Important is Recognition for 3D Scene Flow Estimation in Autonomous Driving Scenarios?},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2017-Octob},
	year = {2017},
	pages = {2593{\textendash}2602},
	abstract = {Existing methods for 3D scene flow estimation often fail in the presence of large displacement or local ambiguities, e.g., at texture-less or reflective surfaces. However, these challenges are omnipresent in dynamic road scenes, which is the focus of this work. Our main contribution is to overcome these 3D motion estimation problems by exploiting recognition. In particular, we investigate the importance of recognition granularity, from coarse 2D bounding box estimates over 2D instance segmentations to fine-grained 3D object part predictions. We compute these cues using CNNs trained on a newly annotated dataset of stereo images and integrate them into a CRF-based model for robust 3D scene flow estimation - an approach we term Instance Scene Flow. We analyze the importance of each recognition cue in an ablation study and observe that the instance segmentation cue is by far strongest, in our setting. We demonstrate the effectiveness of our method on the challenging KITTI 2015 scene flow benchmark where we achieve state-of-the-art performance at the time of submission.},
	isbn = {9781538610329},
	issn = {15505499},
	doi = {10.1109/ICCV.2017.281},
	author = {Behl, Aseem and Omid Hosseini Jafari and Mustikovela, Siva Karthik and Hassan Abu Alhaija and Carsten Rother and Geiger, Andreas}
}
@conference {Behl2017,
	title = {Bounding Boxes, Segmentations and Object Coordinates: How Important is Recognition for 3D Scene Flow Estimation in Autonomous Driving Scenarios?},
	booktitle = {Proceedings of the IEEE International Conference on Computer Vision},
	volume = {2017-Octob},
	year = {2017},
	pages = {2593{\textendash}2602},
	abstract = {Existing methods for 3D scene flow estimation often fail in the presence of large displacement or local ambiguities, e.g., at texture-less or reflective surfaces. However, these challenges are omnipresent in dynamic road scenes, which is the focus of this work. Our main contribution is to overcome these 3D motion estimation problems by exploiting recognition. In particular, we investigate the importance of recognition granularity, from coarse 2D bounding box estimates over 2D instance segmentations to fine-grained 3D object part predictions. We compute these cues using CNNs trained on a newly annotated dataset of stereo images and integrate them into a CRF-based model for robust 3D scene flow estimation - an approach we term Instance Scene Flow. We analyze the importance of each recognition cue in an ablation study and observe that the instance segmentation cue is by far strongest, in our setting. We demonstrate the effectiveness of our method on the challenging KITTI 2015 scene flow benchmark where we achieve state-of-the-art performance at the time of submission.},
	isbn = {9781538610329},
	issn = {15505499},
	doi = {10.1109/ICCV.2017.281},
	author = {Behl, Aseem and Omid Hosseini Jafari and Mustikovela, Siva Karthik and Hassan Abu Alhaija and Carsten Rother and Geiger, Andreas}
}
@conference {Alhaija2015,
	title = {Graphflow{\textemdash}6D large displacement scene flow via graph matching},
	booktitle = {Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)},
	volume = {9358},
	year = {2015},
	pages = {285{\textendash}296},
	abstract = {We present an approach for computing dense scene flow from two large displacement RGB-D images. When dealing with large displacements the crucial step is to estimate the overall motion correctly. While state-of-the-art approaches focus on RGB information to establish guiding correspondences, we explore the power of depth edges. To achieve this, we present a new graph matching technique that brings sparse depth edges into correspondence. An additional contribution is the formulation of a continuous-label energy which is used to densify the sparse graph matching output. We present results on challenging Kinect images, for which we outperform state-of-the-art techniques.},
	isbn = {9783319249469},
	issn = {16113349},
	doi = {10.1007/978-3-319-24947-6_23},
	author = {Hassan Abu Alhaija and Sellent, Anita and Kondermann, Daniel and Carsten Rother}
}
