code: @Override
  	public double calculateObservedDisagreement() {
  		ensureDistanceFunction();
  
  		double result = 0.0;
  		double maxDistance = 1.0;
  		for (ICodingAnnotationItem item : study.getItems()) {
  			Map<Object, Integer> annotationsPerCategory
  					= CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
  
  			for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet())
  				for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) {
  					if (category1.getValue() == null)
  						continue;
  					if (category2.getValue() == null)
  						continue;
  
  					double distance = distanceFunction.measureDistance(study,
  							category1.getKey(), category2.getKey());
  					result += category1.getValue() * category2.getValue()
  							* distance;
  					if (distance > maxDistance)
  						maxDistance = distance;
  				}
  		}
  
  		result /= (double) (maxDistance * study.getItemCount() * study.getRaterCount()
  				* (study.getRaterCount() - 1));
  		return result;
  	}
docs: Calculates the observed inter-rater agreement for the annotation
  study that was passed to the class constructor and the currently
  assigned distance function.
  @throws NullPointerException if the study is null.
  @throws ArithmeticException if the study does not contain any item or
  the number of raters is smaller than 2.