(cls, model, image_data, image_classes, num_classes, num_clusters=30, num_neighbors=20)
return ModelStats(confusion_matrix, images_by_classification, probs_by_image, top_k_images_by_cluster, fit.cluster_centers_)
Create a new ModelStatsDatabase by applying a trained model
        to a collection of images.
Create a new ModelStatsDatabase by applying a trained model
        to a collection of images.
Create a new ModelStatsDatabase by applying a trained model to a collection of images .
def create(cls, model, image_data, image_classes, num_classes, num_clusters=30, num_neighbors=20):
        """
        Create a new ModelStatsDatabase by applying a trained model
        to a collection of images.
        """
        BATCH_SIZE = 2000
        image_data = image_data.astype(np.float32)
        confusion_matrix = np.zeros((num_classes, num_classes))
        images_by_classification = [[[] for _ in xrange(num_classes)] for _ in xrange(num_classes)]
        probs_by_image = None
        fc10_features = []

        for chunk_start in xrange(0, len(image_data), BATCH_SIZE):
            images = image_data[chunk_start:chunk_start + BATCH_SIZE]
            start_time = time.time()
            outputs = model.predict(data=images, output_blobs=["probs_cudanet_out"])
            end_time = time.time()
            _log.info("Processed batch of %i images in %f seconds" %
                     (len(images), end_time - start_time))
            probs = outputs["probs_cudanet_out"]
            if probs_by_image is None:
                probs_by_image = probs
            else:
                probs_by_image = np.concatenate((probs_by_image, probs))
            for (offset, image_probs) in enumerate(probs):
                image_num = chunk_start + offset
                true_class = image_classes[image_num]
                predicted_class = np.argmax(image_probs)
                confusion_matrix[true_class][predicted_class] += 1
                images_by_classification[true_class][predicted_class].append(image_num)
            
            fc10_features.append(outputs["probs_cudanet_out"])
        
        #Code to do top k clusters    
        _log.info("Starting clustering")
        fc10_features = np.vstack(fc10_features)
        _log.info("Features have shape: %s" % str(fc10_features.shape))
        _log.info("Starting K-means")
        from sklearn.cluster import KMeans
        km = KMeans(n_clusters = num_clusters)
        fit = km.fit(fc10_features)
        
        #For each point, get its distance from its cluster center.
        _log.info("Computing distances.")
        dists = np.linalg.norm(fc10_features - fit.cluster_centers_[fit.labels_] , ord=2, axis=1)
        
        #Build a matrix of kNN by point.
        top_k_images_by_cluster = [[] for _ in xrange(num_clusters)]
        for k in xrange(0, num_clusters):
            #Get indexes of elements with this label.
            inds = np.where(fit.labels_ == k)
            
            #Reorder indexes according to closest distance.
            tab = np.column_stack((inds[0],dists[inds]))
            topNeighbors = tab[tab[:,1].argsort()][0:num_neighbors,0]
            top_k_images_by_cluster[k] = map(int, list(topNeighbors))
        
        return ModelStats(confusion_matrix, images_by_classification, probs_by_image, top_k_images_by_cluster, fit.cluster_centers_)
def create ( cls , model , image_data , image_classes , num_classes , num_clusters = 30 , num_neighbors = 20 ) : BATCH_SIZE = 2000 image_data = image_data . astype ( np . float32 ) confusion_matrix = np . zeros ( ( num_classes , num_classes ) ) images_by_classification = [ [ [ ] for _ in xrange ( num_classes ) ] for _ in xrange ( num_classes ) ] probs_by_image = None fc10_features = [ ] for chunk_start in xrange ( 0 , len ( image_data ) , BATCH_SIZE ) : images = image_data [ chunk_start : chunk_start + BATCH_SIZE ] start_time = time . time ( ) outputs = model . predict ( data = images , output_blobs = [ "probs_cudanet_out" ] ) end_time = time . time ( ) _log . info ( "Processed batch of %i images in %f seconds" % ( len ( images ) , end_time - start_time ) ) probs = outputs [ "probs_cudanet_out" ] if probs_by_image is None : probs_by_image = probs else : probs_by_image = np . concatenate ( ( probs_by_image , probs ) ) for ( offset , image_probs ) in enumerate ( probs ) : image_num = chunk_start + offset true_class = image_classes [ image_num ] predicted_class = np . argmax ( image_probs ) confusion_matrix [ true_class ] [ predicted_class ] += 1 images_by_classification [ true_class ] [ predicted_class ] . append ( image_num ) fc10_features . append ( outputs [ "probs_cudanet_out" ] ) #Code to do top k clusters     _log . info ( "Starting clustering" ) fc10_features = np . vstack ( fc10_features ) _log . info ( "Features have shape: %s" % str ( fc10_features . shape ) ) _log . info ( "Starting K-means" ) from sklearn . cluster import KMeans km = KMeans ( n_clusters = num_clusters ) fit = km . fit ( fc10_features ) #For each point, get its distance from its cluster center. _log . info ( "Computing distances." ) dists = np . linalg . norm ( fc10_features - fit . cluster_centers_ [ fit . labels_ ] , ord = 2 , axis = 1 ) #Build a matrix of kNN by point. top_k_images_by_cluster = [ [ ] for _ in xrange ( num_clusters ) ] for k in xrange ( 0 , num_clusters ) : #Get indexes of elements with this label. inds = np . where ( fit . labels_ == k ) #Reorder indexes according to closest distance. tab = np . column_stack ( ( inds [ 0 ] , dists [ inds ] ) ) topNeighbors = tab [ tab [ : , 1 ] . argsort ( ) ] [ 0 : num_neighbors , 0 ] top_k_images_by_cluster [ k ] = map ( int , list ( topNeighbors ) ) return ModelStats ( confusion_matrix , images_by_classification , probs_by_image , top_k_images_by_cluster , fit . cluster_centers_ )