(apks_directory, single_analysis, dynamic_analysis_folder, virus_total_reports_folder,
                       flowdroid_folder, output_folder, noclean_up, package_index_file, classes_index_file,
                       system_commands_file, label, avclass, export_mongodb, export_csv)
Extracts features from a set of samples

    Parameters
    ----------
    :param apks_directory: Folder containing apk files
    :param single_analysis: If an individual features file is generated for each sample
    :param dynamic_analysis_folder: Folder containing dynamic analysis reports
    :param virus_total_reports_folder: Folder containing VirusTotal reports
    :param flowdroid_folder: Folder containing flowdroid reports
    :param output_folder: Folder where features files are saved
    :param noclean_up: If unnecesary files generated are removed
    :param package_index_file: File describing Android API packages
    :param classes_index_file: File describing Android API classes
    :param system_commands_file: File describing Android system commands
    :param label: If provided, all samples are labelled according to this argument
    :param avclass: If avclass is executed to obtain a consensual label for each sample
    :param export_mongodb: Mongodb address to write features to a database
    :param export_csv: If the features extracted are saved into a csv file
Extracts features from a set of samples
Extracts features from a set of samples
def features_extractor(apks_directory, single_analysis, dynamic_analysis_folder, virus_total_reports_folder,
                       flowdroid_folder, output_folder, noclean_up, package_index_file, classes_index_file,
                       system_commands_file, label, avclass, export_mongodb, export_csv):
    """
    Extracts features from a set of samples

    Parameters
    ----------
    :param apks_directory: Folder containing apk files
    :param single_analysis: If an individual features file is generated for each sample
    :param dynamic_analysis_folder: Folder containing dynamic analysis reports
    :param virus_total_reports_folder: Folder containing VirusTotal reports
    :param flowdroid_folder: Folder containing flowdroid reports
    :param output_folder: Folder where features files are saved
    :param noclean_up: If unnecesary files generated are removed
    :param package_index_file: File describing Android API packages
    :param classes_index_file: File describing Android API classes
    :param system_commands_file: File describing Android system commands
    :param label: If provided, all samples are labelled according to this argument
    :param avclass: If avclass is executed to obtain a consensual label for each sample
    :param export_mongodb: Mongodb address to write features to a database
    :param export_csv: If the features extracted are saved into a csv file
    """
    source_directory = str(apks_directory)

    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # Load Android API packages and classes
    global API_PACKAGES_LIST, API_CLASSES_LIST, API_SYSTEM_COMMANDS

    ############################################################
    # READING PACKAGES, CLASSES AND SYSTEM COMMANDS
    ############################################################
    package_file = load_file(str(package_index_file))
    API_PACKAGES_LIST = [x.strip() for x in package_file]

    class_file = load_file(str(classes_index_file))
    API_CLASSES_LIST = [x.strip() for x in class_file]

    system_commands_file = load_file(str(system_commands_file))
    API_SYSTEM_COMMANDS = [x.strip() for x in system_commands_file]
    ############################################################

    ############################################################
    # BUILDING LIST OF APKS
    ############################################################
    apk_list = list_files(source_directory, '*.apk')
    print '[*] Number of APKs:', len(apk_list)
    ############################################################

    ############################################################
    # ANALYSING APKS
    ############################################################
    database = collections.OrderedDict()
    print "ANALYSING APKS..."
    for analyze_apk in tqdm(apk_list):

        # Getting the name of the folder that contains all apks and folders with apks
        base_folder = source_directory.split("/")[-1]

        apk_filename = join_dir(base_folder, analyze_apk.replace(source_directory, ''))
        apk_filename = apk_filename.replace("//", "/")

        apk_name_no_extensions = "".join(apk_filename.split("/")[-1].split(".")[:-1])

        if os.path.isfile(join_dir(output_folder, apk_filename.split("/")[-1].replace('.apk', '-analysis.json'))):
            database[apk_filename.replace('.apk', '')] = json.load(open(join_dir(output_folder, apk_filename.split("/")[-1].
                                                                            replace('.apk', '-analysis.json'))))
            continue

        pre_static_dict = collections.OrderedDict()

        pre_static_dict['Filename'] = apk_filename

        hasher_md5 = hashlib.md5()
        hasher_sha256 = hashlib.sha256()
        hasher_sha1 = hashlib.sha1()
        with open(analyze_apk, 'rb') as afile:
            buf = afile.read()
            hasher_md5.update(buf)
            hasher_sha256.update(buf)
            hasher_sha1.update(buf)

        md5 = hasher_md5.hexdigest()
        sha256 = hasher_sha256.hexdigest()
        sha1 = hasher_sha1.hexdigest()

        pre_static_dict["md5"] = md5
        pre_static_dict["sha256"] = sha256
        pre_static_dict["sha1"] = sha1

        """
        if label is not None:
            pre_static_dict["Label"] = label
        else:
            pre_static_dict["Label"] = "/".join(apk_filename.split("/")[:-1])
        """
        pre_static_dict["VT_positives"] = None

        try:
            androguard_apk_object = apk.APK(analyze_apk)
        except Exception:
            print "ERROR in APK: " + apk_name_no_extensions
            continue

        static_analysis_dict = collections.OrderedDict()
        # Package name
        static_analysis_dict['Package name'] = androguard_apk_object.get_package()

        # Permissions
        static_analysis_dict['Permissions'] = androguard_apk_object.get_permissions()

        # Opcodes
        static_analysis_dict['Opcodes'] = opcodes_analysis(androguard_apk_object)

        # Activities
        try:
            list_activities = androguard_apk_object.get_activities()
        except UnicodeEncodeError:
            list_activities = []

        # Main activity
        static_analysis_dict['Main activity'] = androguard_apk_object.get_main_activity()

        # Receivers
        try:
            list_receivers = androguard_apk_object.get_receivers()
        except UnicodeEncodeError:
            list_receivers = []

        # Services
        try:
            list_services = androguard_apk_object.get_services()
        except UnicodeEncodeError:
            list_services = []

        # API calls and Strings
        list_smali_api_calls, list_smali_strings = read_strings_and_apicalls(analyze_apk, API_PACKAGES_LIST,
                                                                             API_CLASSES_LIST)
        for api_call in list_smali_api_calls.keys():
            new_api_call = '.'.join(api_call.split(".")[:-1])
            if new_api_call in list_smali_api_calls.keys():
                list_smali_api_calls[new_api_call] = list_smali_api_calls[new_api_call] + list_smali_api_calls[api_call]
            else:
                list_smali_api_calls[new_api_call] = list_smali_api_calls[api_call]
                del list_smali_api_calls[api_call]
        static_analysis_dict['API calls'] = list_smali_api_calls
        static_analysis_dict['Strings'] = Counter(filter(None, list_smali_strings))

        # API packages

        API_packages_dict = collections.OrderedDict()
        android_list_packages_lenghts = [len(x.split(".")) for x in API_PACKAGES_LIST]

        list_api_calls_keys = list_smali_api_calls.keys()
        for api_call in list_api_calls_keys:
            score = 0
            package_chosen = None
            for i, package in enumerate(API_PACKAGES_LIST):
                len_package = android_list_packages_lenghts[i]
                if api_call.startswith(package) and len_package > score:
                    score = len_package
                    package_chosen = package
            if package_chosen is not None:
                if not package_chosen in API_packages_dict.keys():
                    API_packages_dict[package_chosen] = list_smali_api_calls[api_call]
                else:
                    API_packages_dict[package_chosen] += list_smali_api_calls[api_call]

        static_analysis_dict['API packages'] = API_packages_dict
        

        # System commands
        list_system_commands = read_system_commands(list_smali_strings, API_SYSTEM_COMMANDS)
        static_analysis_dict['System commands'] = Counter(list_system_commands)

        # Intents
        try:
            static_analysis_dict['Intents'] = intents_analysis(join_dir(analyze_apk.replace('.apk', ''),
                                                                        'AndroidManifest.xml'))
        except:
            static_analysis_dict['Intents'] = {'Failed to extract intents': 0}

        # Intents of activities
        intents_activities = collections.OrderedDict()
        for activity in list_activities:

            
            intents_activities[activity] = check_for_intents(join_dir(analyze_apk.replace('.apk', ''),
                                                                      'AndroidManifest.xml'),
                                                             activity, 'activity')
        static_analysis_dict['Activities'] = intents_activities

        # Intents of services
        intents_services = collections.OrderedDict()
        for service in list_services:
            intents_services[service] = check_for_intents(join_dir(analyze_apk.replace('.apk', ''),
                                                                   'AndroidManifest.xml'),
                                                          service, 'service')
        static_analysis_dict['Services'] = intents_services

        # Intents of receivers
        intents_receivers = collections.OrderedDict()
        for intent in list_receivers:
            intents_receivers[intent] = check_for_intents(join_dir(analyze_apk.replace('.apk', '/'),
                                                                   'AndroidManifest.xml'),
                                                          intent, 'receiver')
        static_analysis_dict['Receivers'] = intents_receivers

        if not noclean_up:
            cleanup(analyze_apk)

        ############################################################
        # READING DYNAMIC ANALYSIS FILES TO INCLUDE IN JSON
        # ONLY THE NAME OF THE FILE IS INCLUDED
        # TODO EACH FILE MUST BE STORED IN A FOLDER NAMED AS THE TOOL USED
        ############################################################
        dynamic_analysis_dict = collections.OrderedDict()

        if dynamic_analysis_folder and isdir(dynamic_analysis_folder):
            dynamic_analysis_folders = [join_dir(dynamic_analysis_folder, x) for x in listdir(str(dynamic_analysis_folder)) if
                                        isdir(join_dir(dynamic_analysis_folder, x))]

            for dynamic_analysis_tool_folder in dynamic_analysis_folders:
                # dynamic_analysis_folder += "/"

                path_to_folder = dynamic_analysis_tool_folder.split("/")
                dynamic_tool_name = filter(None, path_to_folder)[-1]
                for extension in POSSIBLE_DYNAMIC_FILES_EXTENSIONS:
                    if os.path.isfile(join_dir(dynamic_analysis_tool_folder, apk_name_no_extensions + extension)):

                        dynamic_file_name = join_dir(dynamic_analysis_tool_folder, apk_name_no_extensions + extension)

                        # If the file has .json extension, it is added to the global json
                        # If not, the field is filled with the path to the dynamic analysis file
                        if extension == ".json":
                            dynamic_analysis_dict[dynamic_tool_name] = json.load(open(dynamic_file_name))
                        else:
                            dynamic_analysis_dict[dynamic_tool_name] = dynamic_file_name

                        break
        ############################################################
        # READING FLOWDROID ANALYSIS FILES TO INCLUDE IN JSON
        # ONLY THE NAME OF THE FILE IS INCLUDED
        # TODO EACH FILE MUST BE STORED IN A FOLDER NAMED AS THE TOOL USED
        ############################################################
        flowdroid_file = ""
        if flowdroid_folder:

            if isfile(join_dir(flowdroid_folder, apk_name_no_extensions + ".csv")):
                flowdroid_file = join_dir(flowdroid_folder, apk_name_no_extensions + ".csv")
                # static_analysis_dict['FlowDroid'] = flowdroid_field

                data_flowdroid_csv = pd.read_csv(flowdroid_file)

                # Setting column names with the first column
                data_flowdroid_csv.index = data_flowdroid_csv["Sources\\Sinks"]
                if "Sources\\Sinks" in data_flowdroid_csv.columns:        
                    del data_flowdroid_csv["Sources\\Sinks"]

                flowdroid_field = data_flowdroid_csv.to_dict()
                static_analysis_dict['FlowDroid'] = flowdroid_field

        ############################################################
        # READING VIRUSTOTAL FILE TO INCLUDE IN JSON
        ############################################################

        virus_total_dict = collections.OrderedDict()
        if virus_total_reports_folder:
            vt_file_name = join_dir(virus_total_reports_folder, apk_name_no_extensions + ".json")
            if isfile(vt_file_name):
                load_vt_json = load_from_json(vt_file_name)
                virus_total_dict = load_vt_json

                # Saving number of antivirus engines from VirusTotal testing for positive in the pre static section
                pre_static_dict["VT_positives"] = load_vt_json["positives"]
                pre_static_dict["VT_engines"] = len(load_vt_json["scans"].keys())
            else:
                virus_total_dict = ""

        

        ############################################################
        # GETTING AVCLASS LABEL IF VIRUSTOTAL ANALYSIS IS AVAILABLE
        ############################################################

        if virus_total_reports_folder and avclass:
            vt_file_name = join_dir(virus_total_reports_folder, apk_name_no_extensions + ".json")
            if isfile(vt_file_name):
                pre_static_dict["avclass"] = get_avclass_label(vt_file_name)

        ############################################################
        # FILLING APK JSON FIELD
        ############################################################
        apk_total_analysis = OrderedDict([("Pre_static_analysis", pre_static_dict),
                                          ("Static_analysis", static_analysis_dict),
                                          ("Dynamic_analysis", dynamic_analysis_dict),
                                          ("VirusTotal", virus_total_dict)])

        database[apk_filename.replace('.apk', '')] = apk_total_analysis

        ############################################################
        # SAVING ANALYSIS FOR INDIVIDUAL APK WHEN SELECTED
        ############################################################
        if single_analysis:
            # save_single_analysis(join_dir(output_folder, apk_filename.split("/")[-1].
            # replace('.apk', '-analysis.json')),
            #                     apk_total_analysis)

            save_as_json(apk_total_analysis, output_name=join_dir(output_folder, apk_name_no_extensions +
                                                                  "-analysis.json"))

    save_as_json(database, output_name=join_dir(output_folder, OUTPUT_FILE_GLOBAL_JSON))

    ############################################################
    # EXPORTING TO MONGODB
    ############################################################
    if export_mongodb is not None:
        for apk_key in database.keys():
            for call in database[apk_key]["Static_analysis"]["API calls"].keys():
                database[apk_key]["Static_analysis"]["API calls"][call.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["API calls"][call]
                del database[apk_key]["Static_analysis"]["API calls"][call]

            for string in database[apk_key]["Static_analysis"]["Strings"].keys():
                database[apk_key]["Static_analysis"]["Strings"][string.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["Strings"][string]
                del database[apk_key]["Static_analysis"]["Strings"][string]

            for activity in database[apk_key]["Static_analysis"]["Activities"].keys():
                database[apk_key]["Static_analysis"]["Activities"][activity.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["Activities"][activity]
                del database[apk_key]["Static_analysis"]["Activities"][activity]

            for receiver in database[apk_key]["Static_analysis"]["Receivers"].keys():
                database[apk_key]["Static_analysis"]["Receivers"][receiver.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["Receivers"][receiver]
                del database[apk_key]["Static_analysis"]["Receivers"][receiver]

            for intent in database[apk_key]["Static_analysis"]["Intents"].keys():
                database[apk_key]["Static_analysis"]["Intents"][intent.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["Intents"][intent]
                del database[apk_key]["Static_analysis"]["Intents"][intent]

            for package in database[apk_key]["Static_analysis"]["API packages"].keys():
                database[apk_key]["Static_analysis"]["API packages"][package.replace(".", "-")] = \
                    database[apk_key]["Static_analysis"]["API packages"][package]
                del database[apk_key]["Static_analysis"]["API packages"][package]

        client = MongoClient('mongodb://' + export_mongodb)
        # Creating database
        db = client['AndroPyTool_database']

        coll = db['report_' + TIME_EXECUTION]

        coll.insert_one(database).inserted_id

    ############################################################
    # EXPORTING TO CSV
    ############################################################
    if export_csv is not None:
        set_permissions = set()
        set_opcodes = set()
        set_apicalls = set()
        set_systemcommands = set()
        set_intents_activities = set()
        set_intents_services = set()
        set_intents_receivers = set()
        set_api_packages = set()

        for apk_key in tqdm(database.keys()):
            apk_dict = database[apk_key]
            
            if len(apk_key.split("/")) > 1:
                kind = apk_key.split("/")[0]
                hash_app = apk_key.split("/")[1]
            else:
                kind = ""
                hash_app = apk_key

            set_permissions.update(apk_dict["Static_analysis"]["Permissions"])
            set_opcodes.update(apk_dict["Static_analysis"]["Opcodes"])
            set_apicalls.update(apk_dict["Static_analysis"]["API calls"])
            set_systemcommands.update(apk_dict["Static_analysis"]["System commands"])
            
            for activity in apk_dict["Static_analysis"]["Activities"]:
                if apk_dict["Static_analysis"]["Activities"][activity] is not None and \
                    len(apk_dict["Static_analysis"]["Activities"][activity]) > 0:
                    set_intents_activities.update(apk_dict["Static_analysis"]["Activities"][activity])

            for service in apk_dict["Static_analysis"]["Services"]:
                if apk_dict["Static_analysis"]["Services"][service] is not None and \
                    len(apk_dict["Static_analysis"]["Services"][service]) > 0:
                    set_intents_services.update(apk_dict["Static_analysis"]["Services"][service])

            for receiver in apk_dict["Static_analysis"]["Receivers"]:
                if apk_dict["Static_analysis"]["Receivers"][receiver] is not None and \
                    len(apk_dict["Static_analysis"]["Receivers"][receiver]) > 0:
                    set_intents_receivers.update(apk_dict["Static_analysis"]["Receivers"][receiver])

            set_api_packages.update(apk_dict["Static_analysis"]["API packages"])

        list_permissions = [x.replace(" ", "") for x in list(set_permissions)]
        list_opcodes = list(set_opcodes)
        list_apicalls = list(set_apicalls)
        list_systemcommands = list(set_systemcommands)
        list_intents_activities = list(set_intents_activities)
        list_intents_services = list(set_intents_services)
        list_intents_receivers = list(set_intents_receivers)
        list_api_packages = list(set_api_packages)

        for i, apicall in enumerate(list(list_apicalls)):
            list_apicalls[i] = ".".join(apicall.encode('ascii', 'ignore').split(".")[:-1])

        list_apicalls = list(set(list_apicalls))
        

        flowdroid_fields = []
        if flowdroid_folder:
            apk_dict_example = database[database.keys()[0]]
            flowdroid_fields = apk_dict_example["Static_analysis"]["FlowDroid"].keys()
            if "Sources\\Sinks" in flowdroid_fields:
                del flowdroid_fields[flowdroid_fields.index("Sources\\Sinks")]
        
        flowdroid_fields_matrix = [(x, y) for x in flowdroid_fields for y in flowdroid_fields]

        list_rows = []

        rows_permissions = []
        rows_opcodes = []
        rows_apicalls = []
        rows_systemcommands = []
        rows_intents_activities = []
        rows_intents_services = []
        rows_intents_receivers = []
        rows_api_packages = []
                
        for apk_key in tqdm(database.keys()):
            apk_dict = database[apk_key]
            label = None
            if len(apk_key.split("/")) > 1:
                label = apk_key.split("/")[0]
                hash_app = apk_key.split("/")[1]
            else:
                label = ""
                hash_app = apk_key
            
            list_permissions_filled = [0 for x in range(len(list_permissions))]
            for i, item in enumerate(list_permissions):
                if item.replace(" ", "") in apk_dict["Static_analysis"]["Permissions"]:
                    list_permissions_filled[i] = 1

            list_opcodes_filled = [0 for x in range(len(list_opcodes))]
            for i, item in enumerate(list_opcodes):
                if item in apk_dict["Static_analysis"]["Opcodes"]:
                    list_opcodes_filled[i] = apk_dict["Static_analysis"]["Opcodes"][item]

            list_apicalls_filled = [0 for x in range(len(list_apicalls))]
            for i, item in enumerate(list_apicalls):
                if item in apk_dict["Static_analysis"]["API calls"]:
                    list_apicalls_filled[i] = apk_dict["Static_analysis"]["API calls"][item]

            list_systemcommands_filled = [0 for x in range(len(list_systemcommands))]
            for i, item in enumerate(list_systemcommands):
                if item in apk_dict["Static_analysis"]["System commands"]:
                    list_systemcommands_filled[i] = apk_dict["Static_analysis"]["System commands"][item]

            list_intents_activities_filled = [0 for x in range(len(list_intents_activities))]
            for i, item in enumerate(list_intents_activities):
                if item in apk_dict["Static_analysis"]["Activities"]:
                    list_intents_activities_filled[i] = 1

            list_intents_services_filled = [0 for x in range(len(list_intents_services))]
            for i, item in enumerate(list_intents_services):
                if item in apk_dict["Static_analysis"]["Services"]:
                    list_intents_services_filled[i] = 1

            list_intents_receivers_filled = [0 for x in range(len(list_intents_receivers))]
            for i, item in enumerate(list_intents_receivers):
                if item in apk_dict["Static_analysis"]["Receivers"]:
                    list_intents_receivers_filled[i] = 1

            list_api_packages_filled = [0 for x in range(len(list_api_packages))]
            for i, item in enumerate(list_api_packages):
                if item in apk_dict["Static_analysis"]["API packages"]:
                    list_intents_receivers_filled[i] = 1

            flowdroid_fields_matrix_filled = [0 for x in range(len(flowdroid_fields_matrix))]
            flow_df = pd.read_csv(join_dir(flowdroid_folder, hash_app + ".csv"))
            flow_df = flow_df.set_index("Sources\Sinks")
            for i, item in enumerate(flowdroid_fields_matrix):
                source, sink = item[0], item[1]
                flowdroid_fields_matrix_filled[i] = flow_df[source][sink]
            complete_row = [label] + list_permissions_filled + list_opcodes_filled + list_apicalls_filled + \
                        list_systemcommands_filled + list_intents_activities_filled + \
                        list_intents_services_filled + list_intents_receivers_filled + list_api_packages_filled + \
                        flowdroid_fields_matrix_filled

            rows_permissions.append(list_permissions_filled)
            rows_opcodes.append(list_opcodes_filled)
            rows_apicalls.append(list_apicalls_filled)
            rows_systemcommands.append(list_systemcommands_filled)
            rows_intents_activities.append(list_intents_activities_filled)
            rows_intents_services.append(list_intents_services_filled)
            rows_intents_receivers.append(list_intents_receivers_filled)
            rows_api_packages.append(list_api_packages_filled)
            list_rows.append(complete_row)

        list_permissions = ["PERMISSION-" + x for x in list(list_permissions)]
        list_opcodes = ["OPCODE-" + x for x in list(list_opcodes)]
        list_apicalls = ["APICALL-" + x for x in list(list_apicalls)]
        list_systemcommands = ["SYSTEMCOMMAND-" + x for x in list(list_systemcommands)]
        list_intents_activities = ["ACTIVITY-" + x for x in list(list_intents_activities)]
        list_intents_services = ["SERVICE-" + x for x in list(list_intents_services)]
        list_intents_receivers = ["RECEIVER-" + x for x in list(list_intents_receivers)]
        list_api_packages = ["APIPACKAGE-" + x for x in list(list_api_packages)]

        flowdroid_fields_matrix_strings = ["FLOWDROID-" + x[0] + "-" + x[1] for x in flowdroid_fields_matrix]

        complete_list_fields = ["label"] + list_permissions + list_opcodes + list_apicalls + \
                       list_systemcommands + list_intents_activities + list_intents_services + list_intents_receivers + \
                       list_api_packages + flowdroid_fields_matrix_strings

        with open(output_folder + "/" +  export_csv, 'wb') as csv_file:

            csvwriter = csv.writer(csv_file, delimiter=",")
            csvwriter.writerow(complete_list_fields)
            print "WRITING CSV FILE..."
            for row in tqdm(list_rows):
                csvwriter.writerow(row)
def features_extractor ( apks_directory , single_analysis , dynamic_analysis_folder , virus_total_reports_folder , flowdroid_folder , output_folder , noclean_up , package_index_file , classes_index_file , system_commands_file , label , avclass , export_mongodb , export_csv ) : source_directory = str ( apks_directory ) if not os . path . exists ( output_folder ) : os . makedirs ( output_folder ) # Load Android API packages and classes global API_PACKAGES_LIST , API_CLASSES_LIST , API_SYSTEM_COMMANDS ############################################################ # READING PACKAGES, CLASSES AND SYSTEM COMMANDS ############################################################ package_file = load_file ( str ( package_index_file ) ) API_PACKAGES_LIST = [ x . strip ( ) for x in package_file ] class_file = load_file ( str ( classes_index_file ) ) API_CLASSES_LIST = [ x . strip ( ) for x in class_file ] system_commands_file = load_file ( str ( system_commands_file ) ) API_SYSTEM_COMMANDS = [ x . strip ( ) for x in system_commands_file ] ############################################################ ############################################################ # BUILDING LIST OF APKS ############################################################ apk_list = list_files ( source_directory , '*.apk' ) print '[*] Number of APKs:' , len ( apk_list ) ############################################################ ############################################################ # ANALYSING APKS ############################################################ database = collections . OrderedDict ( ) print "ANALYSING APKS..." for analyze_apk in tqdm ( apk_list ) : # Getting the name of the folder that contains all apks and folders with apks base_folder = source_directory . split ( "/" ) [ - 1 ] apk_filename = join_dir ( base_folder , analyze_apk . replace ( source_directory , '' ) ) apk_filename = apk_filename . replace ( "//" , "/" ) apk_name_no_extensions = "" . join ( apk_filename . split ( "/" ) [ - 1 ] . split ( "." ) [ : - 1 ] ) if os . path . isfile ( join_dir ( output_folder , apk_filename . split ( "/" ) [ - 1 ] . replace ( '.apk' , '-analysis.json' ) ) ) : database [ apk_filename . replace ( '.apk' , '' ) ] = json . load ( open ( join_dir ( output_folder , apk_filename . split ( "/" ) [ - 1 ] . replace ( '.apk' , '-analysis.json' ) ) ) ) continue pre_static_dict = collections . OrderedDict ( ) pre_static_dict [ 'Filename' ] = apk_filename hasher_md5 = hashlib . md5 ( ) hasher_sha256 = hashlib . sha256 ( ) hasher_sha1 = hashlib . sha1 ( ) with open ( analyze_apk , 'rb' ) as afile : buf = afile . read ( ) hasher_md5 . update ( buf ) hasher_sha256 . update ( buf ) hasher_sha1 . update ( buf ) md5 = hasher_md5 . hexdigest ( ) sha256 = hasher_sha256 . hexdigest ( ) sha1 = hasher_sha1 . hexdigest ( ) pre_static_dict [ "md5" ] = md5 pre_static_dict [ "sha256" ] = sha256 pre_static_dict [ "sha1" ] = sha1 """
        if label is not None:
            pre_static_dict["Label"] = label
        else:
            pre_static_dict["Label"] = "/".join(apk_filename.split("/")[:-1])
        """ pre_static_dict [ "VT_positives" ] = None try : androguard_apk_object = apk . APK ( analyze_apk ) except Exception : print "ERROR in APK: " + apk_name_no_extensions continue static_analysis_dict = collections . OrderedDict ( ) # Package name static_analysis_dict [ 'Package name' ] = androguard_apk_object . get_package ( ) # Permissions static_analysis_dict [ 'Permissions' ] = androguard_apk_object . get_permissions ( ) # Opcodes static_analysis_dict [ 'Opcodes' ] = opcodes_analysis ( androguard_apk_object ) # Activities try : list_activities = androguard_apk_object . get_activities ( ) except UnicodeEncodeError : list_activities = [ ] # Main activity static_analysis_dict [ 'Main activity' ] = androguard_apk_object . get_main_activity ( ) # Receivers try : list_receivers = androguard_apk_object . get_receivers ( ) except UnicodeEncodeError : list_receivers = [ ] # Services try : list_services = androguard_apk_object . get_services ( ) except UnicodeEncodeError : list_services = [ ] # API calls and Strings list_smali_api_calls , list_smali_strings = read_strings_and_apicalls ( analyze_apk , API_PACKAGES_LIST , API_CLASSES_LIST ) for api_call in list_smali_api_calls . keys ( ) : new_api_call = '.' . join ( api_call . split ( "." ) [ : - 1 ] ) if new_api_call in list_smali_api_calls . keys ( ) : list_smali_api_calls [ new_api_call ] = list_smali_api_calls [ new_api_call ] + list_smali_api_calls [ api_call ] else : list_smali_api_calls [ new_api_call ] = list_smali_api_calls [ api_call ] del list_smali_api_calls [ api_call ] static_analysis_dict [ 'API calls' ] = list_smali_api_calls static_analysis_dict [ 'Strings' ] = Counter ( filter ( None , list_smali_strings ) ) # API packages API_packages_dict = collections . OrderedDict ( ) android_list_packages_lenghts = [ len ( x . split ( "." ) ) for x in API_PACKAGES_LIST ] list_api_calls_keys = list_smali_api_calls . keys ( ) for api_call in list_api_calls_keys : score = 0 package_chosen = None for i , package in enumerate ( API_PACKAGES_LIST ) : len_package = android_list_packages_lenghts [ i ] if api_call . startswith ( package ) and len_package > score : score = len_package package_chosen = package if package_chosen is not None : if not package_chosen in API_packages_dict . keys ( ) : API_packages_dict [ package_chosen ] = list_smali_api_calls [ api_call ] else : API_packages_dict [ package_chosen ] += list_smali_api_calls [ api_call ] static_analysis_dict [ 'API packages' ] = API_packages_dict # System commands list_system_commands = read_system_commands ( list_smali_strings , API_SYSTEM_COMMANDS ) static_analysis_dict [ 'System commands' ] = Counter ( list_system_commands ) # Intents try : static_analysis_dict [ 'Intents' ] = intents_analysis ( join_dir ( analyze_apk . replace ( '.apk' , '' ) , 'AndroidManifest.xml' ) ) except : static_analysis_dict [ 'Intents' ] = { 'Failed to extract intents' : 0 } # Intents of activities intents_activities = collections . OrderedDict ( ) for activity in list_activities : intents_activities [ activity ] = check_for_intents ( join_dir ( analyze_apk . replace ( '.apk' , '' ) , 'AndroidManifest.xml' ) , activity , 'activity' ) static_analysis_dict [ 'Activities' ] = intents_activities # Intents of services intents_services = collections . OrderedDict ( ) for service in list_services : intents_services [ service ] = check_for_intents ( join_dir ( analyze_apk . replace ( '.apk' , '' ) , 'AndroidManifest.xml' ) , service , 'service' ) static_analysis_dict [ 'Services' ] = intents_services # Intents of receivers intents_receivers = collections . OrderedDict ( ) for intent in list_receivers : intents_receivers [ intent ] = check_for_intents ( join_dir ( analyze_apk . replace ( '.apk' , '/' ) , 'AndroidManifest.xml' ) , intent , 'receiver' ) static_analysis_dict [ 'Receivers' ] = intents_receivers if not noclean_up : cleanup ( analyze_apk ) ############################################################ # READING DYNAMIC ANALYSIS FILES TO INCLUDE IN JSON # ONLY THE NAME OF THE FILE IS INCLUDED # TODO EACH FILE MUST BE STORED IN A FOLDER NAMED AS THE TOOL USED ############################################################ dynamic_analysis_dict = collections . OrderedDict ( ) if dynamic_analysis_folder and isdir ( dynamic_analysis_folder ) : dynamic_analysis_folders = [ join_dir ( dynamic_analysis_folder , x ) for x in listdir ( str ( dynamic_analysis_folder ) ) if isdir ( join_dir ( dynamic_analysis_folder , x ) ) ] for dynamic_analysis_tool_folder in dynamic_analysis_folders : # dynamic_analysis_folder += "/" path_to_folder = dynamic_analysis_tool_folder . split ( "/" ) dynamic_tool_name = filter ( None , path_to_folder ) [ - 1 ] for extension in POSSIBLE_DYNAMIC_FILES_EXTENSIONS : if os . path . isfile ( join_dir ( dynamic_analysis_tool_folder , apk_name_no_extensions + extension ) ) : dynamic_file_name = join_dir ( dynamic_analysis_tool_folder , apk_name_no_extensions + extension ) # If the file has .json extension, it is added to the global json # If not, the field is filled with the path to the dynamic analysis file if extension == ".json" : dynamic_analysis_dict [ dynamic_tool_name ] = json . load ( open ( dynamic_file_name ) ) else : dynamic_analysis_dict [ dynamic_tool_name ] = dynamic_file_name break ############################################################ # READING FLOWDROID ANALYSIS FILES TO INCLUDE IN JSON # ONLY THE NAME OF THE FILE IS INCLUDED # TODO EACH FILE MUST BE STORED IN A FOLDER NAMED AS THE TOOL USED ############################################################ flowdroid_file = "" if flowdroid_folder : if isfile ( join_dir ( flowdroid_folder , apk_name_no_extensions + ".csv" ) ) : flowdroid_file = join_dir ( flowdroid_folder , apk_name_no_extensions + ".csv" ) # static_analysis_dict['FlowDroid'] = flowdroid_field data_flowdroid_csv = pd . read_csv ( flowdroid_file ) # Setting column names with the first column data_flowdroid_csv . index = data_flowdroid_csv [ "Sources\\Sinks" ] if "Sources\\Sinks" in data_flowdroid_csv . columns : del data_flowdroid_csv [ "Sources\\Sinks" ] flowdroid_field = data_flowdroid_csv . to_dict ( ) static_analysis_dict [ 'FlowDroid' ] = flowdroid_field ############################################################ # READING VIRUSTOTAL FILE TO INCLUDE IN JSON ############################################################ virus_total_dict = collections . OrderedDict ( ) if virus_total_reports_folder : vt_file_name = join_dir ( virus_total_reports_folder , apk_name_no_extensions + ".json" ) if isfile ( vt_file_name ) : load_vt_json = load_from_json ( vt_file_name ) virus_total_dict = load_vt_json # Saving number of antivirus engines from VirusTotal testing for positive in the pre static section pre_static_dict [ "VT_positives" ] = load_vt_json [ "positives" ] pre_static_dict [ "VT_engines" ] = len ( load_vt_json [ "scans" ] . keys ( ) ) else : virus_total_dict = "" ############################################################ # GETTING AVCLASS LABEL IF VIRUSTOTAL ANALYSIS IS AVAILABLE ############################################################ if virus_total_reports_folder and avclass : vt_file_name = join_dir ( virus_total_reports_folder , apk_name_no_extensions + ".json" ) if isfile ( vt_file_name ) : pre_static_dict [ "avclass" ] = get_avclass_label ( vt_file_name ) ############################################################ # FILLING APK JSON FIELD ############################################################ apk_total_analysis = OrderedDict ( [ ( "Pre_static_analysis" , pre_static_dict ) , ( "Static_analysis" , static_analysis_dict ) , ( "Dynamic_analysis" , dynamic_analysis_dict ) , ( "VirusTotal" , virus_total_dict ) ] ) database [ apk_filename . replace ( '.apk' , '' ) ] = apk_total_analysis ############################################################ # SAVING ANALYSIS FOR INDIVIDUAL APK WHEN SELECTED ############################################################ if single_analysis : # save_single_analysis(join_dir(output_folder, apk_filename.split("/")[-1]. # replace('.apk', '-analysis.json')), #                     apk_total_analysis) save_as_json ( apk_total_analysis , output_name = join_dir ( output_folder , apk_name_no_extensions + "-analysis.json" ) ) save_as_json ( database , output_name = join_dir ( output_folder , OUTPUT_FILE_GLOBAL_JSON ) ) ############################################################ # EXPORTING TO MONGODB ############################################################ if export_mongodb is not None : for apk_key in database . keys ( ) : for call in database [ apk_key ] [ "Static_analysis" ] [ "API calls" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "API calls" ] [ call . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "API calls" ] [ call ] del database [ apk_key ] [ "Static_analysis" ] [ "API calls" ] [ call ] for string in database [ apk_key ] [ "Static_analysis" ] [ "Strings" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "Strings" ] [ string . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "Strings" ] [ string ] del database [ apk_key ] [ "Static_analysis" ] [ "Strings" ] [ string ] for activity in database [ apk_key ] [ "Static_analysis" ] [ "Activities" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "Activities" ] [ activity . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "Activities" ] [ activity ] del database [ apk_key ] [ "Static_analysis" ] [ "Activities" ] [ activity ] for receiver in database [ apk_key ] [ "Static_analysis" ] [ "Receivers" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "Receivers" ] [ receiver . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "Receivers" ] [ receiver ] del database [ apk_key ] [ "Static_analysis" ] [ "Receivers" ] [ receiver ] for intent in database [ apk_key ] [ "Static_analysis" ] [ "Intents" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "Intents" ] [ intent . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "Intents" ] [ intent ] del database [ apk_key ] [ "Static_analysis" ] [ "Intents" ] [ intent ] for package in database [ apk_key ] [ "Static_analysis" ] [ "API packages" ] . keys ( ) : database [ apk_key ] [ "Static_analysis" ] [ "API packages" ] [ package . replace ( "." , "-" ) ] = database [ apk_key ] [ "Static_analysis" ] [ "API packages" ] [ package ] del database [ apk_key ] [ "Static_analysis" ] [ "API packages" ] [ package ] client = MongoClient ( 'mongodb://' + export_mongodb ) # Creating database db = client [ 'AndroPyTool_database' ] coll = db [ 'report_' + TIME_EXECUTION ] coll . insert_one ( database ) . inserted_id ############################################################ # EXPORTING TO CSV ############################################################ if export_csv is not None : set_permissions = set ( ) set_opcodes = set ( ) set_apicalls = set ( ) set_systemcommands = set ( ) set_intents_activities = set ( ) set_intents_services = set ( ) set_intents_receivers = set ( ) set_api_packages = set ( ) for apk_key in tqdm ( database . keys ( ) ) : apk_dict = database [ apk_key ] if len ( apk_key . split ( "/" ) ) > 1 : kind = apk_key . split ( "/" ) [ 0 ] hash_app = apk_key . split ( "/" ) [ 1 ] else : kind = "" hash_app = apk_key set_permissions . update ( apk_dict [ "Static_analysis" ] [ "Permissions" ] ) set_opcodes . update ( apk_dict [ "Static_analysis" ] [ "Opcodes" ] ) set_apicalls . update ( apk_dict [ "Static_analysis" ] [ "API calls" ] ) set_systemcommands . update ( apk_dict [ "Static_analysis" ] [ "System commands" ] ) for activity in apk_dict [ "Static_analysis" ] [ "Activities" ] : if apk_dict [ "Static_analysis" ] [ "Activities" ] [ activity ] is not None and len ( apk_dict [ "Static_analysis" ] [ "Activities" ] [ activity ] ) > 0 : set_intents_activities . update ( apk_dict [ "Static_analysis" ] [ "Activities" ] [ activity ] ) for service in apk_dict [ "Static_analysis" ] [ "Services" ] : if apk_dict [ "Static_analysis" ] [ "Services" ] [ service ] is not None and len ( apk_dict [ "Static_analysis" ] [ "Services" ] [ service ] ) > 0 : set_intents_services . update ( apk_dict [ "Static_analysis" ] [ "Services" ] [ service ] ) for receiver in apk_dict [ "Static_analysis" ] [ "Receivers" ] : if apk_dict [ "Static_analysis" ] [ "Receivers" ] [ receiver ] is not None and len ( apk_dict [ "Static_analysis" ] [ "Receivers" ] [ receiver ] ) > 0 : set_intents_receivers . update ( apk_dict [ "Static_analysis" ] [ "Receivers" ] [ receiver ] ) set_api_packages . update ( apk_dict [ "Static_analysis" ] [ "API packages" ] ) list_permissions = [ x . replace ( " " , "" ) for x in list ( set_permissions ) ] list_opcodes = list ( set_opcodes ) list_apicalls = list ( set_apicalls ) list_systemcommands = list ( set_systemcommands ) list_intents_activities = list ( set_intents_activities ) list_intents_services = list ( set_intents_services ) list_intents_receivers = list ( set_intents_receivers ) list_api_packages = list ( set_api_packages ) for i , apicall in enumerate ( list ( list_apicalls ) ) : list_apicalls [ i ] = "." . join ( apicall . encode ( 'ascii' , 'ignore' ) . split ( "." ) [ : - 1 ] ) list_apicalls = list ( set ( list_apicalls ) ) flowdroid_fields = [ ] if flowdroid_folder : apk_dict_example = database [ database . keys ( ) [ 0 ] ] flowdroid_fields = apk_dict_example [ "Static_analysis" ] [ "FlowDroid" ] . keys ( ) if "Sources\\Sinks" in flowdroid_fields : del flowdroid_fields [ flowdroid_fields . index ( "Sources\\Sinks" ) ] flowdroid_fields_matrix = [ ( x , y ) for x in flowdroid_fields for y in flowdroid_fields ] list_rows = [ ] rows_permissions = [ ] rows_opcodes = [ ] rows_apicalls = [ ] rows_systemcommands = [ ] rows_intents_activities = [ ] rows_intents_services = [ ] rows_intents_receivers = [ ] rows_api_packages = [ ] for apk_key in tqdm ( database . keys ( ) ) : apk_dict = database [ apk_key ] label = None if len ( apk_key . split ( "/" ) ) > 1 : label = apk_key . split ( "/" ) [ 0 ] hash_app = apk_key . split ( "/" ) [ 1 ] else : label = "" hash_app = apk_key list_permissions_filled = [ 0 for x in range ( len ( list_permissions ) ) ] for i , item in enumerate ( list_permissions ) : if item . replace ( " " , "" ) in apk_dict [ "Static_analysis" ] [ "Permissions" ] : list_permissions_filled [ i ] = 1 list_opcodes_filled = [ 0 for x in range ( len ( list_opcodes ) ) ] for i , item in enumerate ( list_opcodes ) : if item in apk_dict [ "Static_analysis" ] [ "Opcodes" ] : list_opcodes_filled [ i ] = apk_dict [ "Static_analysis" ] [ "Opcodes" ] [ item ] list_apicalls_filled = [ 0 for x in range ( len ( list_apicalls ) ) ] for i , item in enumerate ( list_apicalls ) : if item in apk_dict [ "Static_analysis" ] [ "API calls" ] : list_apicalls_filled [ i ] = apk_dict [ "Static_analysis" ] [ "API calls" ] [ item ] list_systemcommands_filled = [ 0 for x in range ( len ( list_systemcommands ) ) ] for i , item in enumerate ( list_systemcommands ) : if item in apk_dict [ "Static_analysis" ] [ "System commands" ] : list_systemcommands_filled [ i ] = apk_dict [ "Static_analysis" ] [ "System commands" ] [ item ] list_intents_activities_filled = [ 0 for x in range ( len ( list_intents_activities ) ) ] for i , item in enumerate ( list_intents_activities ) : if item in apk_dict [ "Static_analysis" ] [ "Activities" ] : list_intents_activities_filled [ i ] = 1 list_intents_services_filled = [ 0 for x in range ( len ( list_intents_services ) ) ] for i , item in enumerate ( list_intents_services ) : if item in apk_dict [ "Static_analysis" ] [ "Services" ] : list_intents_services_filled [ i ] = 1 list_intents_receivers_filled = [ 0 for x in range ( len ( list_intents_receivers ) ) ] for i , item in enumerate ( list_intents_receivers ) : if item in apk_dict [ "Static_analysis" ] [ "Receivers" ] : list_intents_receivers_filled [ i ] = 1 list_api_packages_filled = [ 0 for x in range ( len ( list_api_packages ) ) ] for i , item in enumerate ( list_api_packages ) : if item in apk_dict [ "Static_analysis" ] [ "API packages" ] : list_intents_receivers_filled [ i ] = 1 flowdroid_fields_matrix_filled = [ 0 for x in range ( len ( flowdroid_fields_matrix ) ) ] flow_df = pd . read_csv ( join_dir ( flowdroid_folder , hash_app + ".csv" ) ) flow_df = flow_df . set_index ( "Sources\Sinks" ) for i , item in enumerate ( flowdroid_fields_matrix ) : source , sink = item [ 0 ] , item [ 1 ] flowdroid_fields_matrix_filled [ i ] = flow_df [ source ] [ sink ] complete_row = [ label ] + list_permissions_filled + list_opcodes_filled + list_apicalls_filled + list_systemcommands_filled + list_intents_activities_filled + list_intents_services_filled + list_intents_receivers_filled + list_api_packages_filled + flowdroid_fields_matrix_filled rows_permissions . append ( list_permissions_filled ) rows_opcodes . append ( list_opcodes_filled ) rows_apicalls . append ( list_apicalls_filled ) rows_systemcommands . append ( list_systemcommands_filled ) rows_intents_activities . append ( list_intents_activities_filled ) rows_intents_services . append ( list_intents_services_filled ) rows_intents_receivers . append ( list_intents_receivers_filled ) rows_api_packages . append ( list_api_packages_filled ) list_rows . append ( complete_row ) list_permissions = [ "PERMISSION-" + x for x in list ( list_permissions ) ] list_opcodes = [ "OPCODE-" + x for x in list ( list_opcodes ) ] list_apicalls = [ "APICALL-" + x for x in list ( list_apicalls ) ] list_systemcommands = [ "SYSTEMCOMMAND-" + x for x in list ( list_systemcommands ) ] list_intents_activities = [ "ACTIVITY-" + x for x in list ( list_intents_activities ) ] list_intents_services = [ "SERVICE-" + x for x in list ( list_intents_services ) ] list_intents_receivers = [ "RECEIVER-" + x for x in list ( list_intents_receivers ) ] list_api_packages = [ "APIPACKAGE-" + x for x in list ( list_api_packages ) ] flowdroid_fields_matrix_strings = [ "FLOWDROID-" + x [ 0 ] + "-" + x [ 1 ] for x in flowdroid_fields_matrix ] complete_list_fields = [ "label" ] + list_permissions + list_opcodes + list_apicalls + list_systemcommands + list_intents_activities + list_intents_services + list_intents_receivers + list_api_packages + flowdroid_fields_matrix_strings with open ( output_folder + "/" + export_csv , 'wb' ) as csv_file : csvwriter = csv . writer ( csv_file , delimiter = "," ) csvwriter . writerow ( complete_list_fields ) print "WRITING CSV FILE..." for row in tqdm ( list_rows ) : csvwriter . writerow ( row )