Module monk.pytorch.finetune.level_4_evaluation_base

Expand source code
from pytorch.finetune.imports import *
from system.imports import *

from pytorch.finetune.level_3_training_base import finetune_training


class finetune_evaluation(finetune_training):
    '''
    Bae class for external validation and inferencing

    Args:
        verbose (int): Set verbosity levels
                        0 - Print Nothing
                        1 - Print desired details
    '''
    def __init__(self, verbose=1):
        super().__init__(verbose=verbose);


    ###############################################################################################################################################
    def set_evaluation_final(self):
        '''
        Main function for external validation post training

        Args:
            None

        Returns:
            float: Accuracy in percentage
            dict: Class based accuracy in percentage
        '''
        self.custom_print("Testing");
        self.system_dict["testing"]["status"] = False;
        if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
            pbar=tqdm(total=len(self.system_dict["local"]["data_loaders"]["test"]));

        running_corrects = 0
        class_dict = {};
        for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]] = {};
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"] = 0;
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"] = 0;

        for inputs, labels in self.system_dict["local"]["data_loaders"]["test"]:
            if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
                pbar.update();

            inputs = inputs.to(self.system_dict["local"]["device"]);
            labels = labels.to(self.system_dict["local"]["device"]);

            outputs = self.system_dict["local"]["model"](inputs)
            _, preds = torch.max(outputs, 1)

            label = int(labels.data.cpu().numpy())
            pred = int(preds.data.cpu().numpy())
            class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_images"] += 1;
            if(label == pred):
                class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_correct"] += 1;
            running_corrects += torch.sum(preds == labels.data)

        accuracy = running_corrects.double() / len(self.system_dict["local"]["data_loaders"]["test"].dataset)

        self.custom_print("");
        self.custom_print("    Result");
        self.custom_print("        class based accuracies");
        for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
            self.custom_print("            {}. {} - {} %".format(i, self.system_dict["dataset"]["params"]["classes"][i], 
                class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100));
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["accuracy(%)"] = class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100;
        self.custom_print("        total images:            {}".format(len(self.system_dict["local"]["data_loaders"]["test"])));
        self.custom_print("        num correct predictions: {}".format(int(running_corrects.cpu().numpy())));
        self.custom_print("        Average accuracy (%):    {}".format(accuracy.cpu().numpy()*100));

        self.system_dict["testing"]["num_images"] = len(self.system_dict["local"]["data_loaders"]["test"]);
        self.system_dict["testing"]["num_correct_predictions"] = int(running_corrects.cpu().numpy());
        self.system_dict["testing"]["percentage_accuracy"] = accuracy.cpu().numpy()*100;
        self.system_dict["testing"]["class_accuracy"] = class_dict
        self.system_dict["testing"]["status"] = True;
        self.custom_print("");
        return accuracy.cpu().numpy()*100, class_dict;
    ###############################################################################################################################################






    ###############################################################################################################################################
    def set_prediction_final(self, img_name=False, img_dir=False, return_raw=False):
        '''
        Main function for external inferencing on single image or folder of images post training

        Args:
            img_name (str): path to image
            img_dir (str): path to folders containing images. 
                            (Optional)
            return_raw (bool): If True, then output dictionary contains image probability for every class in the set.
                                Else, only the most probable class score is returned back.
                                

        Returns:
            float: Accuracy in percentage
            dict: Inference output
                   1) Image name
                   2) Predicted class
                   3) Predicted score
        '''
        self.custom_print("Prediction");

        if(not self.system_dict["dataset"]["params"]["input_size"]):
            msg = "Input Size not set for experiment.\n";
            msg += "Tip: Use update_input_size";
            raise ConstraintError(msg);

        self.system_dict = set_transform_test(self.system_dict);


        if(not self.system_dict["dataset"]["params"]["classes"]):
            msg = "Class information unavailabe.\n";
            msg += "Labels returned - Indexes instead of classes";
            ConstraintWarning(msg);


        if(img_name):
            self.custom_print("    Image name:         {}".format(img_name));

            label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

            self.custom_print("    Predicted class:      {}".format(label));
            self.custom_print("    Predicted score:      {}".format(score));
            tmp = {};
            tmp["img_name"] = img_name;
            tmp["predicted_class"] = label;
            tmp["score"] = score;
            if(return_raw):
                tmp["raw"] = raw_output;
            self.custom_print("");                
            return tmp;

        if(img_dir):
            output = [];
            self.custom_print("    Dir path:           {}".format(img_dir));
            img_list = os.listdir(img_dir);
            self.custom_print("    Total Images:       {}".format(len(img_list)));
            self.custom_print("Processing Images");
            if(self.system_dict["verbose"]):
                pbar = tqdm(total=len(img_list));


            for i in range(len(img_list)):
                if(self.system_dict["verbose"]):
                    pbar.update();

                img_name = img_dir + "/" + img_list[i];
                
                label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

                tmp = {};
                tmp["img_name"] = img_list[i];
                tmp["predicted_class"] = label;
                tmp["score"] = score;
                if(return_raw):
                    tmp["raw"] = raw_output;
                output.append(tmp);
            self.custom_print("");

            return output
    ###############################################################################################################################################

Classes

class finetune_evaluation (verbose=1)

Bae class for external validation and inferencing

Args

verbose : int
Set verbosity levels 0 - Print Nothing 1 - Print desired details
Expand source code
class finetune_evaluation(finetune_training):
    '''
    Bae class for external validation and inferencing

    Args:
        verbose (int): Set verbosity levels
                        0 - Print Nothing
                        1 - Print desired details
    '''
    def __init__(self, verbose=1):
        super().__init__(verbose=verbose);


    ###############################################################################################################################################
    def set_evaluation_final(self):
        '''
        Main function for external validation post training

        Args:
            None

        Returns:
            float: Accuracy in percentage
            dict: Class based accuracy in percentage
        '''
        self.custom_print("Testing");
        self.system_dict["testing"]["status"] = False;
        if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
            pbar=tqdm(total=len(self.system_dict["local"]["data_loaders"]["test"]));

        running_corrects = 0
        class_dict = {};
        for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]] = {};
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"] = 0;
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"] = 0;

        for inputs, labels in self.system_dict["local"]["data_loaders"]["test"]:
            if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
                pbar.update();

            inputs = inputs.to(self.system_dict["local"]["device"]);
            labels = labels.to(self.system_dict["local"]["device"]);

            outputs = self.system_dict["local"]["model"](inputs)
            _, preds = torch.max(outputs, 1)

            label = int(labels.data.cpu().numpy())
            pred = int(preds.data.cpu().numpy())
            class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_images"] += 1;
            if(label == pred):
                class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_correct"] += 1;
            running_corrects += torch.sum(preds == labels.data)

        accuracy = running_corrects.double() / len(self.system_dict["local"]["data_loaders"]["test"].dataset)

        self.custom_print("");
        self.custom_print("    Result");
        self.custom_print("        class based accuracies");
        for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
            self.custom_print("            {}. {} - {} %".format(i, self.system_dict["dataset"]["params"]["classes"][i], 
                class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100));
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["accuracy(%)"] = class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100;
        self.custom_print("        total images:            {}".format(len(self.system_dict["local"]["data_loaders"]["test"])));
        self.custom_print("        num correct predictions: {}".format(int(running_corrects.cpu().numpy())));
        self.custom_print("        Average accuracy (%):    {}".format(accuracy.cpu().numpy()*100));

        self.system_dict["testing"]["num_images"] = len(self.system_dict["local"]["data_loaders"]["test"]);
        self.system_dict["testing"]["num_correct_predictions"] = int(running_corrects.cpu().numpy());
        self.system_dict["testing"]["percentage_accuracy"] = accuracy.cpu().numpy()*100;
        self.system_dict["testing"]["class_accuracy"] = class_dict
        self.system_dict["testing"]["status"] = True;
        self.custom_print("");
        return accuracy.cpu().numpy()*100, class_dict;
    ###############################################################################################################################################






    ###############################################################################################################################################
    def set_prediction_final(self, img_name=False, img_dir=False, return_raw=False):
        '''
        Main function for external inferencing on single image or folder of images post training

        Args:
            img_name (str): path to image
            img_dir (str): path to folders containing images. 
                            (Optional)
            return_raw (bool): If True, then output dictionary contains image probability for every class in the set.
                                Else, only the most probable class score is returned back.
                                

        Returns:
            float: Accuracy in percentage
            dict: Inference output
                   1) Image name
                   2) Predicted class
                   3) Predicted score
        '''
        self.custom_print("Prediction");

        if(not self.system_dict["dataset"]["params"]["input_size"]):
            msg = "Input Size not set for experiment.\n";
            msg += "Tip: Use update_input_size";
            raise ConstraintError(msg);

        self.system_dict = set_transform_test(self.system_dict);


        if(not self.system_dict["dataset"]["params"]["classes"]):
            msg = "Class information unavailabe.\n";
            msg += "Labels returned - Indexes instead of classes";
            ConstraintWarning(msg);


        if(img_name):
            self.custom_print("    Image name:         {}".format(img_name));

            label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

            self.custom_print("    Predicted class:      {}".format(label));
            self.custom_print("    Predicted score:      {}".format(score));
            tmp = {};
            tmp["img_name"] = img_name;
            tmp["predicted_class"] = label;
            tmp["score"] = score;
            if(return_raw):
                tmp["raw"] = raw_output;
            self.custom_print("");                
            return tmp;

        if(img_dir):
            output = [];
            self.custom_print("    Dir path:           {}".format(img_dir));
            img_list = os.listdir(img_dir);
            self.custom_print("    Total Images:       {}".format(len(img_list)));
            self.custom_print("Processing Images");
            if(self.system_dict["verbose"]):
                pbar = tqdm(total=len(img_list));


            for i in range(len(img_list)):
                if(self.system_dict["verbose"]):
                    pbar.update();

                img_name = img_dir + "/" + img_list[i];
                
                label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

                tmp = {};
                tmp["img_name"] = img_list[i];
                tmp["predicted_class"] = label;
                tmp["score"] = score;
                if(return_raw):
                    tmp["raw"] = raw_output;
                output.append(tmp);
            self.custom_print("");

            return output

Ancestors

  • pytorch.finetune.level_3_training_base.finetune_training
  • pytorch.finetune.level_2_model_base.finetune_model
  • pytorch.finetune.level_1_dataset_base.finetune_dataset
  • system.base_class.system

Methods

def set_evaluation_final(self)

Main function for external validation post training

Args

None
 

Returns

float
Accuracy in percentage
dict
Class based accuracy in percentage
Expand source code
def set_evaluation_final(self):
    '''
    Main function for external validation post training

    Args:
        None

    Returns:
        float: Accuracy in percentage
        dict: Class based accuracy in percentage
    '''
    self.custom_print("Testing");
    self.system_dict["testing"]["status"] = False;
    if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
        pbar=tqdm(total=len(self.system_dict["local"]["data_loaders"]["test"]));

    running_corrects = 0
    class_dict = {};
    for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
        class_dict[self.system_dict["dataset"]["params"]["classes"][i]] = {};
        class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"] = 0;
        class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"] = 0;

    for inputs, labels in self.system_dict["local"]["data_loaders"]["test"]:
        if(self.system_dict["training"]["settings"]["display_progress_realtime"] and self.system_dict["verbose"]):
            pbar.update();

        inputs = inputs.to(self.system_dict["local"]["device"]);
        labels = labels.to(self.system_dict["local"]["device"]);

        outputs = self.system_dict["local"]["model"](inputs)
        _, preds = torch.max(outputs, 1)

        label = int(labels.data.cpu().numpy())
        pred = int(preds.data.cpu().numpy())
        class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_images"] += 1;
        if(label == pred):
            class_dict[self.system_dict["dataset"]["params"]["classes"][label]]["num_correct"] += 1;
        running_corrects += torch.sum(preds == labels.data)

    accuracy = running_corrects.double() / len(self.system_dict["local"]["data_loaders"]["test"].dataset)

    self.custom_print("");
    self.custom_print("    Result");
    self.custom_print("        class based accuracies");
    for i in range(len(self.system_dict["dataset"]["params"]["classes"])):
        self.custom_print("            {}. {} - {} %".format(i, self.system_dict["dataset"]["params"]["classes"][i], 
            class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100));
        class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["accuracy(%)"] = class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_correct"]/class_dict[self.system_dict["dataset"]["params"]["classes"][i]]["num_images"]*100;
    self.custom_print("        total images:            {}".format(len(self.system_dict["local"]["data_loaders"]["test"])));
    self.custom_print("        num correct predictions: {}".format(int(running_corrects.cpu().numpy())));
    self.custom_print("        Average accuracy (%):    {}".format(accuracy.cpu().numpy()*100));

    self.system_dict["testing"]["num_images"] = len(self.system_dict["local"]["data_loaders"]["test"]);
    self.system_dict["testing"]["num_correct_predictions"] = int(running_corrects.cpu().numpy());
    self.system_dict["testing"]["percentage_accuracy"] = accuracy.cpu().numpy()*100;
    self.system_dict["testing"]["class_accuracy"] = class_dict
    self.system_dict["testing"]["status"] = True;
    self.custom_print("");
    return accuracy.cpu().numpy()*100, class_dict;
def set_prediction_final(self, img_name=False, img_dir=False, return_raw=False)

Main function for external inferencing on single image or folder of images post training

Args

img_name : str
path to image
img_dir : str
path to folders containing images. (Optional)
return_raw : bool
If True, then output dictionary contains image probability for every class in the set. Else, only the most probable class score is returned back.

Returns

float
Accuracy in percentage
dict
Inference output 1) Image name 2) Predicted class 3) Predicted score
Expand source code
def set_prediction_final(self, img_name=False, img_dir=False, return_raw=False):
    '''
    Main function for external inferencing on single image or folder of images post training

    Args:
        img_name (str): path to image
        img_dir (str): path to folders containing images. 
                        (Optional)
        return_raw (bool): If True, then output dictionary contains image probability for every class in the set.
                            Else, only the most probable class score is returned back.
                            

    Returns:
        float: Accuracy in percentage
        dict: Inference output
               1) Image name
               2) Predicted class
               3) Predicted score
    '''
    self.custom_print("Prediction");

    if(not self.system_dict["dataset"]["params"]["input_size"]):
        msg = "Input Size not set for experiment.\n";
        msg += "Tip: Use update_input_size";
        raise ConstraintError(msg);

    self.system_dict = set_transform_test(self.system_dict);


    if(not self.system_dict["dataset"]["params"]["classes"]):
        msg = "Class information unavailabe.\n";
        msg += "Labels returned - Indexes instead of classes";
        ConstraintWarning(msg);


    if(img_name):
        self.custom_print("    Image name:         {}".format(img_name));

        label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

        self.custom_print("    Predicted class:      {}".format(label));
        self.custom_print("    Predicted score:      {}".format(score));
        tmp = {};
        tmp["img_name"] = img_name;
        tmp["predicted_class"] = label;
        tmp["score"] = score;
        if(return_raw):
            tmp["raw"] = raw_output;
        self.custom_print("");                
        return tmp;

    if(img_dir):
        output = [];
        self.custom_print("    Dir path:           {}".format(img_dir));
        img_list = os.listdir(img_dir);
        self.custom_print("    Total Images:       {}".format(len(img_list)));
        self.custom_print("Processing Images");
        if(self.system_dict["verbose"]):
            pbar = tqdm(total=len(img_list));


        for i in range(len(img_list)):
            if(self.system_dict["verbose"]):
                pbar.update();

            img_name = img_dir + "/" + img_list[i];
            
            label, score, raw_output = process_single(img_name, return_raw, self.system_dict);

            tmp = {};
            tmp["img_name"] = img_list[i];
            tmp["predicted_class"] = label;
            tmp["score"] = score;
            if(return_raw):
                tmp["raw"] = raw_output;
            output.append(tmp);
        self.custom_print("");

        return output